Skip to main content

roder_core/
runtime.rs

1use std::collections::HashMap;
2use std::path::PathBuf;
3use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
4use std::sync::{Arc, Weak};
5
6use anyhow::Context;
7use futures::StreamExt;
8use futures::future::{AbortHandle, Abortable, BoxFuture, try_join_all};
9use roder_api::catalog::{
10    EDIT_TOOL_EDIT, EDIT_TOOL_PATCH, PROVIDER_GEMINI, REASONING_NONE, REASONING_ULTRA,
11    built_in_model_profile, built_in_model_profile_for_provider, lookup_model,
12    model_supports_reasoning_effort,
13};
14use roder_api::context::PolicyGate;
15use roder_api::events::*;
16use roder_api::extension::ExtensionRegistry;
17use roder_api::inference::{
18    AgentInferenceRequest, HostedWebSearchConfig, HostedWebSearchMode, InferenceEngine,
19    InferenceEvent, InferenceTurnContext, InstructionBundle, ModelHarnessProfile,
20    ModelSchemaPolicy, ModelSelection, OutputConfig, ProviderTurnCleanup, ReasoningConfig,
21    RuntimeHints, RuntimeProfile, TokenUsage, ToolCallCompleted, ToolSearchConfig,
22    ToolSearchConfigOverlay, finish_reason_from_stop_reason,
23};
24use roder_api::inference_routing::{InferenceRoutingOutcome, ModelSelectionMode};
25use roder_api::lifecycle::{
26    LifecycleMetricsSnapshot, TurnCleanupOwnership, TurnCleanupState, TurnLifecycleReason,
27    TurnLifecycleRecord, TurnLifecycleSnapshot, TurnLifecycleState, turn_lifecycle_snapshot,
28};
29use roder_api::policy_mode::{PolicyDecision, PolicyMode};
30use roder_api::reliability::{
31    ReliabilityContext, ReliabilityDetails, ReliabilityErrorClass, ReliabilityLimitDecision,
32    ReliabilityLimitRecorded, ReliabilityRequestPolicy, ReliabilityRetryDecision,
33    ReliabilityRetryRecorded, provider_retry_delay_ms,
34};
35use roder_api::remote_runner::{
36    RemoteRunnerProvider, RemoteRunnerSession, RemoteWorkspace, RunnerDestination,
37    RunnerSessionState, ThreadRunnerBinding,
38};
39use roder_api::subagents::SubagentDefinition;
40use roder_api::teams::{
41    TeamId, TeamMailboxMessage, TeamMailboxMessageKind, TeamMemberDescriptor, TeamMemberRole,
42    TeamMemberStatus,
43};
44use roder_api::thread::{
45    ThreadItemEvent, ThreadItemEventKind, ThreadMetadata, ThreadSnapshot, ThreadStore,
46    ThreadUsageMetadata, is_synthetic_event_thread_id, validate_thread_workspace,
47};
48use roder_api::tools::{ToolCall, ToolChoice, ToolExecutionContext, ToolRegistry, ToolResult};
49use roder_api::transcript::{
50    AssistantMessage, ErrorRecord, InputImage, ReasoningSummary, ToolCallRecord, ToolResultRecord,
51    TranscriptItem, UserMessage,
52};
53use roder_sandbox::ScopedFilesystem;
54use roder_sandbox::process::LocalProcessRunner;
55use roder_skills::{SkillRegistry, SkillRegistryOptions};
56use time::{Duration, OffsetDateTime};
57use tokio::sync::{Mutex, Notify, RwLock, oneshot};
58
59mod codex_v2;
60
61use crate::artifacts::{
62    ContextArtifactStore as FilesystemContextArtifactStore, default_context_artifact_dir,
63};
64use crate::bus::EventBus;
65use crate::dynamic_workflows::{
66    DynamicWorkflowEffortProfile, RuntimeDynamicWorkflowConfig, WorkflowTriggerDecision,
67    classify_workflow_trigger, ultracode_reasoning_level_for_model,
68};
69use crate::fake_provider::FakeInferenceEngine;
70use crate::goals::RuntimeGoalController;
71use crate::inference_routing::{
72    InferenceRoutingRequest, RuntimeInferenceRouterConfig, collect_inference_routing_candidates,
73    route_inference_selection, transcript_failure_count_since,
74};
75use crate::instructions::{
76    apply_agent_swarm_mode, apply_codex_multi_agent_mode, apply_model_instruction_overlay,
77    apply_plan_mode, apply_runtime_profile, apply_task_ledger_required,
78    apply_thread_developer_instructions, apply_turn_developer_context,
79};
80use crate::policy_gate::DefaultPolicyGate;
81use crate::reliability::{
82    ReliabilityLimitHit, RuntimeReliabilityConfig, TurnReliabilityState,
83    provider_stream_retry_cause,
84};
85pub use crate::speed_policy::RuntimeSpeedPolicyConfig;
86use crate::speed_policy::{SpeedPolicyState, reasoning_from_decision};
87use crate::subagent_traces::{RuntimeAgentSwarmProgressSink, RuntimeSubagentTraceSink};
88use crate::teams::{TeamManager, TeamMemberStartRequest, TeamStartRequest, TeamState};
89use crate::thread_item_cache::{ThreadItemCache, ThreadItemCacheEntry};
90use crate::verification_gate::VerificationGateState;
91
92const MAX_TOOL_ROUNDS_PER_TURN: usize = 1024;
93/// Injected when `continue_on_failure_limit` turns a consecutive-tool-failure
94/// limit into a continuation, or as the empty-tool-call persistence nudge, so
95/// the model keeps working instead of ending the turn early.
96const RELIABILITY_CONTINUATION_PROMPT: &str = "Verify the task is fully complete. If any part remains unfinished or unverified, keep working using the available tools — try an alternative approach if one is failing. Only stop once the solution is complete and verified; if it is already complete, restate your final answer.";
97/// Capacity of the runtime event broadcast ring buffer. The TUI drains this on
98/// its render loop, which during an active turn only wakes at the ~6 FPS status
99/// animation cadence (backend events do not wake the input poll), so up to
100/// ~166ms of events buffer between drains. A reasoning-high provider that runs
101/// its whole tool loop inside one turn (e.g. claude-code) streams a firehose of
102/// `InferenceEventReceived` deltas plus per-step tool/thinking events; the old
103/// 1024-slot buffer overflowed in those windows and silently dropped tool and
104/// thinking rows from the live view. Sized for generous headroom across bursts
105/// and brief render stalls.
106const EVENT_BUS_CAPACITY: usize = 16_384;
107const FINAL_ANSWER_PHASE: &str = "final_answer";
108pub(crate) const TASK_LEDGER_TOOL_NAME: &str = "task_ledger.update";
109const TASK_LEDGER_COMPLETION_REMINDER_LIMIT: u8 = 2;
110const TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS: u64 = 180;
111const TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT: u8 = 1;
112pub(crate) const MIN_CHILD_DEADLINE_SECONDS: u64 = 2;
113const MODEL_PROFILE_TRACE_KIND: &str = "model_profile_segment";
114const MODEL_SWITCH_SUMMARY_PREFIX: &str = "Model switch summary:";
115const PROVIDER_CLEANUP_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5);
116
117#[derive(Clone, Copy, Debug, Eq, PartialEq)]
118enum InferenceTimeoutAction {
119    ScoreableCheckpoint,
120    Finalization,
121}
122
123#[derive(Debug, Clone)]
124pub struct RuntimeConfig {
125    pub default_provider: String,
126    pub default_model: String,
127    pub reasoning: Option<String>,
128    pub auto_compact_token_limit: Option<u32>,
129    pub file_backed_dynamic_context: bool,
130    pub hosted_web_search: HostedWebSearchConfig,
131    pub tool_search: ToolSearchConfig,
132    pub provider_tool_search: HashMap<String, ToolSearchConfigOverlay>,
133    pub model_tool_search: HashMap<String, ToolSearchConfigOverlay>,
134    pub model_edit_tools: HashMap<String, String>,
135    pub model_parallel_tool_calls: HashMap<String, bool>,
136    pub model_profiles: HashMap<String, ModelHarnessProfile>,
137    pub tool_allowlist: Vec<String>,
138    /// Seconds a host-executed external tool call may stay unresolved before it fails with a timeout error.
139    pub external_tool_timeout_seconds: u64,
140    pub command_shell: String,
141    pub workspace: Option<String>,
142    pub policy_mode: PolicyMode,
143    /// Whether agent-swarm mode is active (roadmap 104). When on, the runtime
144    /// injects the swarm reminder into each turn's developer instructions so any
145    /// client benefits, not just the TUI.
146    pub agent_swarm_mode: bool,
147    pub runtime_profile: RuntimeProfile,
148    pub inference_router: RuntimeInferenceRouterConfig,
149    pub speed_policy: RuntimeSpeedPolicyConfig,
150    pub dynamic_workflows: RuntimeDynamicWorkflowConfig,
151    pub reliability: RuntimeReliabilityConfig,
152    /// Positive wall-clock limit for a turn in any runtime profile. `None` or zero is unbounded.
153    pub turn_deadline_seconds: Option<u64>,
154    pub remote_runner_destination: Option<RunnerDestination>,
155    pub team_data_dir: Option<PathBuf>,
156    pub roadmap_data_dir: Option<PathBuf>,
157    pub media_generation: crate::media_generation::RuntimeMediaGenerationConfig,
158}
159
160impl Default for RuntimeConfig {
161    fn default() -> Self {
162        Self {
163            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
164            default_model: "mock".to_string(),
165            reasoning: None,
166            auto_compact_token_limit: None,
167            file_backed_dynamic_context: true,
168            hosted_web_search: HostedWebSearchConfig::cached(),
169            tool_search: ToolSearchConfig::default(),
170            provider_tool_search: HashMap::new(),
171            model_tool_search: HashMap::new(),
172            model_edit_tools: HashMap::new(),
173            model_parallel_tool_calls: HashMap::new(),
174            model_profiles: HashMap::new(),
175            tool_allowlist: Vec::new(),
176            external_tool_timeout_seconds: DEFAULT_EXTERNAL_TOOL_TIMEOUT_SECONDS,
177            command_shell: roder_api::command_shell::default_command_shell(),
178            workspace: None,
179            policy_mode: PolicyMode::Default,
180            agent_swarm_mode: false,
181            runtime_profile: RuntimeProfile::Interactive,
182            inference_router: RuntimeInferenceRouterConfig::default(),
183            speed_policy: RuntimeSpeedPolicyConfig::default(),
184            dynamic_workflows: RuntimeDynamicWorkflowConfig::default(),
185            reliability: RuntimeReliabilityConfig::default(),
186            turn_deadline_seconds: None,
187            remote_runner_destination: None,
188            team_data_dir: None,
189            roadmap_data_dir: None,
190            media_generation: crate::media_generation::RuntimeMediaGenerationConfig::default(),
191        }
192    }
193}
194
195#[derive(Debug, Clone)]
196pub struct StartTurnRequest {
197    pub thread_id: ThreadId,
198    pub message: String,
199    pub images: Vec<InputImage>,
200    pub provider_override: Option<String>,
201    pub model_override: Option<String>,
202    pub reasoning_override: Option<String>,
203    pub workspace: String,
204    pub instructions: InstructionBundle,
205    /**
206     * Per-turn developer-authority context for this turn's InstructionBundle.
207     * Applies to every inference round of the turn, is never written to
208     * thread state, and does not carry over to later turns.
209     */
210    pub developer_context: Option<String>,
211    pub task_ledger_required: bool,
212}
213
214#[derive(Debug, Clone)]
215pub struct CreateThreadRequest {
216    pub title: Option<String>,
217    pub workspace: String,
218    pub workspace_id: Option<String>,
219    pub root_id: Option<String>,
220    pub provider: Option<String>,
221    pub model: Option<String>,
222    pub selection_mode: Option<ModelSelectionMode>,
223    /// Per-thread tool filter applied on top of the runtime allowlist. Empty = no filtering.
224    pub tool_allowlist: Vec<String>,
225    /// Host-supplied instructions added to the developer slot of every turn's inference request.
226    pub developer_instructions: Option<String>,
227    /// Host-executed tool specs advertised to the model on every turn of this thread.
228    pub external_tools: Vec<roder_api::tools::ToolSpec>,
229    /// Explicit remote-runner binding for the thread's native coding tools.
230    pub runner: Option<ThreadRunnerSelection>,
231}
232
233/**
234 * Thread-level remote-runner selection. The destination config is persisted
235 * with the thread, so secrets must reach the provider through its
236 * environment, not this config.
237 */
238#[derive(Debug, Clone)]
239pub struct ThreadRunnerSelection {
240    pub provider_id: String,
241    pub config: serde_json::Value,
242    /// Absolute path on the runner used as the thread's coding-tool workspace root.
243    pub workspace: String,
244    /**
245     * Extra absolute runner paths file reads may resolve under, beyond
246     * `workspace`. Writes and the working directory stay confined to
247     * `workspace`.
248     */
249    pub read_roots: Vec<String>,
250}
251
252#[derive(Debug, Clone, PartialEq, Eq)]
253pub struct PendingPlanExit {
254    pub thread_id: ThreadId,
255    pub turn_id: TurnId,
256    pub request_id: String,
257    pub target_mode: PolicyMode,
258    pub plan_summary: Option<String>,
259    pub next_steps: Vec<String>,
260    pub requested_at: OffsetDateTime,
261    pub expires_at: Option<OffsetDateTime>,
262}
263
264pub(crate) struct PendingToolApproval {
265    pub(crate) thread_id: ThreadId,
266    pub(crate) turn_id: TurnId,
267    pub(crate) tool_id: String,
268    pub(crate) tool_name: String,
269    pub(crate) call: roder_api::tools::ToolCall,
270    pub(crate) tx: oneshot::Sender<bool>,
271}
272
273pub(crate) struct PendingUserInput {
274    pub(crate) thread_id: ThreadId,
275    pub(crate) turn_id: TurnId,
276    pub(crate) tx: oneshot::Sender<serde_json::Value>,
277}
278
279/// Host answer to an external tool call delivered via `tools/resolve`.
280#[derive(Debug, Clone, PartialEq, Eq)]
281pub struct ExternalToolResolution {
282    pub output: String,
283    pub is_error: bool,
284}
285
286pub(crate) struct PendingExternalToolCall {
287    pub(crate) thread_id: ThreadId,
288    pub(crate) turn_id: TurnId,
289    pub(crate) tool_id: String,
290    pub(crate) tool_name: String,
291    pub(crate) tx: oneshot::Sender<ExternalToolResolution>,
292}
293
294#[derive(Clone)]
295struct ActiveTurnHandle {
296    thread_id: ThreadId,
297    abort: AbortHandle,
298    steers: Arc<Mutex<Vec<QueuedTurnSteer>>>,
299    drain: Arc<TurnDrainHandle>,
300}
301
302/// Tracks a task after it has been interrupted and removed from the interactive
303/// active-turn map. This lets users start a follow-up turn immediately while a
304/// bounded runtime shutdown can still await the original task's cleanup.
305struct TurnDrainHandle {
306    thread_id: ThreadId,
307    interrupt_requested: AtomicBool,
308    interrupt_reason: Mutex<Option<TurnLifecycleReason>>,
309    completed: AtomicBool,
310    completed_notify: Notify,
311}
312
313/// Result of a bounded runtime drain. `Clean` proves the runtime's own turn
314/// tasks have reached their cleanup paths; provider-specific child-process
315/// reaping remains an explicit provider capability.
316#[derive(Debug, Clone, PartialEq, Eq)]
317pub enum RuntimeDrainOutcome {
318    Clean {
319        interrupted_turn_ids: Vec<TurnId>,
320    },
321    DeadlineExceeded {
322        interrupted_turn_ids: Vec<TurnId>,
323        remaining_turn_ids: Vec<TurnId>,
324    },
325    /// At least one lifecycle transition initiated by this drain could not be
326    /// durably appended. The runtime still made its best cleanup attempt, but
327    /// callers must not claim a persisted terminal state as proof of recovery.
328    PersistenceFailed {
329        interrupted_turn_ids: Vec<TurnId>,
330        remaining_turn_ids: Vec<TurnId>,
331    },
332}
333
334#[derive(Clone)]
335struct QueuedTurnSteer {
336    message: UserMessage,
337    mailbox_ack: Option<MailboxDeliveryAck>,
338}
339
340#[derive(Clone)]
341struct MailboxDeliveryAck {
342    team_id: TeamId,
343    message_ids: Vec<String>,
344}
345
346#[derive(Clone)]
347struct InheritedTurnContext {
348    workspace: String,
349    instructions: InstructionBundle,
350    developer_context: Option<String>,
351}
352
353#[derive(Debug, Clone, Default, PartialEq, Eq)]
354pub struct ThreadActivity {
355    pub active_turn_id: Option<TurnId>,
356    pub active_flags: Vec<String>,
357}
358
359/// Per-thread settings persisted at thread creation and applied to every turn.
360#[derive(Debug, Clone, Default)]
361pub(crate) struct ThreadTurnOverrides {
362    pub(crate) tool_allowlist: Vec<String>,
363    pub(crate) developer_instructions: Option<String>,
364    pub(crate) external_tools: Vec<roder_api::tools::ToolSpec>,
365}
366
367#[derive(Debug, Clone, Copy, PartialEq, Eq)]
368pub(crate) enum TurnRunOutcome {
369    Completed,
370    Stopped,
371}
372
373impl PendingPlanExit {
374    pub fn new(
375        thread_id: ThreadId,
376        turn_id: TurnId,
377        request_id: String,
378        target_mode: PolicyMode,
379        plan_summary: Option<String>,
380        next_steps: Vec<String>,
381    ) -> Self {
382        let requested_at = OffsetDateTime::now_utc();
383        Self {
384            thread_id,
385            turn_id,
386            request_id,
387            target_mode,
388            plan_summary,
389            next_steps,
390            requested_at,
391            expires_at: Some(requested_at + default_plan_exit_timeout()),
392        }
393    }
394
395    pub fn is_expired(&self, now: OffsetDateTime) -> bool {
396        self.expires_at.is_some_and(|expires_at| now >= expires_at)
397    }
398}
399
400pub fn default_plan_exit_timeout() -> Duration {
401    Duration::minutes(10)
402}
403
404pub const DEFAULT_EXTERNAL_TOOL_TIMEOUT_SECONDS: u64 = 300;
405
406fn format_mailbox_messages(team: &TeamState, messages: &[TeamMailboxMessage]) -> String {
407    messages
408        .iter()
409        .map(|message| {
410            let recipient = team
411                .members
412                .iter()
413                .find(|member| member.id == message.to_member_id)
414                .map(canonical_team_member_path)
415                .unwrap_or_else(|| format!("/root/{}", message.to_member_id));
416            let sender = message
417                .from_member_id
418                .as_deref()
419                .and_then(|id| team.members.iter().find(|member| member.id == id))
420                .map(canonical_team_member_path)
421                .unwrap_or_else(|| "/root".to_string());
422            let message_type = match message.kind {
423                TeamMailboxMessageKind::Message => "MESSAGE",
424                TeamMailboxMessageKind::NewTask => "NEW_TASK",
425                TeamMailboxMessageKind::FinalAnswer => "FINAL_ANSWER",
426            };
427            format!(
428                "Message Type: {message_type}\nTask name: {recipient}\nSender: {sender}\nPayload:\n{}",
429                message.text
430            )
431        })
432        .collect::<Vec<_>>()
433        .join("\n\n")
434}
435
436fn canonical_team_member_path(member: &TeamMemberDescriptor) -> String {
437    if let Some(agent_path) = member
438        .agent_path
439        .as_deref()
440        .filter(|path| *path == "/root" || path.starts_with("/root/"))
441    {
442        return agent_path.to_string();
443    }
444    if member.role == TeamMemberRole::Lead {
445        return "/root".to_string();
446    }
447    member
448        .task_name
449        .as_deref()
450        .filter(|name| !name.trim().is_empty())
451        .map(|name| format!("/root/{}", name.trim().trim_matches('/')))
452        .unwrap_or_else(|| format!("/root/{}", member.id))
453}
454
455fn is_codex_v2_team(team: &TeamState) -> bool {
456    team.members.iter().any(|member| {
457        member
458            .model
459            .as_deref()
460            .is_some_and(|model| model_supports_reasoning_effort(model, REASONING_ULTRA))
461    })
462}
463
464fn runtime_local_team_view(
465    mut team: TeamState,
466    active_thread_ids: &std::collections::HashSet<ThreadId>,
467) -> TeamState {
468    for member in &mut team.members {
469        if member.status == TeamMemberStatus::Running
470            && !active_thread_ids.contains(&member.thread_id)
471        {
472            member.status = TeamMemberStatus::Interrupted;
473            member.current_turn_id = None;
474        }
475    }
476    team
477}
478
479type RunnerToolExecutionKey = (String, String);
480type RunnerToolExecutionMutex = Mutex<()>;
481type RunnerToolExecutionLockRegistry =
482    Mutex<HashMap<RunnerToolExecutionKey, Weak<RunnerToolExecutionMutex>>>;
483
484pub struct Runtime {
485    pub bus: EventBus,
486    pub registry: ExtensionRegistry,
487    config: RwLock<RuntimeConfig>,
488    pending_plan_exit: RwLock<Option<PendingPlanExit>>,
489    pub(crate) pending_tool_approvals: Mutex<HashMap<String, PendingToolApproval>>,
490    pub(crate) pending_user_inputs: Mutex<HashMap<String, PendingUserInput>>,
491    pub(crate) pending_external_tool_calls: Mutex<HashMap<String, PendingExternalToolCall>>,
492    /// Serializes turn admission with shutdown quiescing. A drain takes this
493    /// gate before flipping `accepting_turns` and snapshotting active work so a
494    /// turn that observed the old flag cannot be inserted after the snapshot.
495    turn_admission: Mutex<()>,
496    active_turns: RwLock<HashMap<TurnId, ActiveTurnHandle>>,
497    turn_drains: RwLock<HashMap<TurnId, Arc<TurnDrainHandle>>>,
498    /// Provider cleanup handles register synchronously through the inference
499    /// tool-executor context. They are intentionally separate from active turn
500    /// admission so an interrupted turn can remain drainable after a follow-up
501    /// turn starts on the same thread.
502    provider_turn_cleanups: std::sync::Mutex<HashMap<TurnId, Arc<dyn ProviderTurnCleanup>>>,
503    accepting_turns: AtomicBool,
504    /// Monotonic counter used by bounded drains to surface lifecycle-state
505    /// persistence failures without making an individual provider turn fail.
506    lifecycle_persistence_failures: AtomicUsize,
507    lifecycle_shutdown_drains: AtomicUsize,
508    lifecycle_clean_shutdowns: AtomicUsize,
509    lifecycle_deadline_exceeded: AtomicUsize,
510    lifecycle_persistence_failed_drains: AtomicUsize,
511    lifecycle_restart_reconciliations: AtomicUsize,
512    lifecycle_shutdown_drain_duration_ms_total: AtomicUsize,
513    provider_cleanup_confirmed: AtomicUsize,
514    provider_cleanup_timed_out: AtomicUsize,
515    provider_cleanup_unknown: AtomicUsize,
516    active_turns_changed: Notify,
517    active_turn_selections: RwLock<HashMap<TurnId, ModelSelectionMode>>,
518    active_turn_contexts: RwLock<HashMap<TurnId, InheritedTurnContext>>,
519    /// Process-local execution boundary. Local/CLI runtimes default to true;
520    /// hosted runtime pools set this false so missing runner metadata fails
521    /// closed instead of exposing the host workspace.
522    allow_local_workspaces: AtomicBool,
523    // Spawn-time live authority retained for every reusable turn of a long-lived teammate.
524    // This stays process-local because developer_context is explicitly volatile and must never
525    // be persisted to thread state.
526    team_member_turn_contexts: Mutex<HashMap<ThreadId, InheritedTurnContext>>,
527    workspace: PathBuf,
528    teams: TeamManager,
529    agent_team_spawn_lock: Mutex<()>,
530    pub(crate) roadmaps: Mutex<roder_roadmap::RoadmapRuntime>,
531    pub(crate) goals: Arc<RuntimeGoalController>,
532    context_artifacts: roder_api::artifacts::ContextArtifactStore,
533    pub(crate) thread_store: Option<Arc<dyn ThreadStore>>,
534    thread_item_cache: Mutex<ThreadItemCache>,
535    pub(crate) tool_registry: ToolRegistry,
536    media_generation: Arc<crate::media_generation::MediaGenerationService>,
537    pub(crate) skills: RwLock<SkillRegistry>,
538    /// Lazily-started bounded dispatch of emitted events to registry
539    /// `EventSink`s (process extensions etc.); see `event_sink_dispatch`.
540    event_sink_dispatcher: tokio::sync::OnceCell<crate::event_sink_dispatch::EventSinkDispatcher>,
541    pub(crate) compaction_hysteresis: std::sync::Mutex<HashMap<ThreadId, u32>>,
542    /// Per-thread agent-swarm mode overrides (roadmap 104). A thread present in
543    /// this map uses its stored value; absent threads fall back to the
544    /// runtime-global `RuntimeConfig.agent_swarm_mode` default. This mirrors the
545    /// team per-member policy-mode override idiom so swarm mode is per-thread
546    /// like the other `thread/*` operations, without a separate runtime.
547    agent_swarm_modes: RwLock<HashMap<ThreadId, bool>>,
548    /**
549     * Live remote-runner sessions keyed by thread. Each per-thread mutex is
550     * held across provider initialization, making the first workspace-tool
551     * access a singleflight while allowing unrelated threads to initialize in
552     * parallel. Failed initialization is shared with current waiters, then
553     * evicted so a later tool call can retry.
554     */
555    runner_sessions: Arc<Mutex<HashMap<ThreadId, Arc<RunnerSessionSlot>>>>,
556    /**
557     * Outer tool-call locks keyed by the actual remote session, not the
558     * per-thread destination id. Hosted runtimes are tenant-scoped, so this
559     * serializes threads that share one sandbox without coupling different
560     * tenants or independent runner sessions.
561     */
562    runner_tool_execution_locks: RunnerToolExecutionLockRegistry,
563}
564
565#[derive(Clone)]
566struct CachedRunnerSession {
567    destination: RunnerDestination,
568    session: Arc<dyn RemoteRunnerSession>,
569}
570
571struct RunnerSessionSlot {
572    provider_id: String,
573    destination_id: String,
574    state: Mutex<RunnerSessionSlotState>,
575    initialized: Notify,
576}
577
578enum RunnerSessionSlotState {
579    Empty,
580    Initializing,
581    Ready(CachedRunnerSession),
582    Failed(Arc<str>),
583}
584
585impl RunnerSessionSlot {
586    fn empty(destination: &RunnerDestination) -> Self {
587        Self {
588            provider_id: destination.provider_id.clone(),
589            destination_id: destination.id.clone(),
590            state: Mutex::new(RunnerSessionSlotState::Empty),
591            initialized: Notify::new(),
592        }
593    }
594
595    fn ready(session: CachedRunnerSession) -> Self {
596        Self {
597            provider_id: session.destination.provider_id.clone(),
598            destination_id: session.destination.id.clone(),
599            state: Mutex::new(RunnerSessionSlotState::Ready(session)),
600            initialized: Notify::new(),
601        }
602    }
603
604    fn matches(&self, destination: &RunnerDestination) -> bool {
605        self.provider_id == destination.provider_id && self.destination_id == destination.id
606    }
607}
608
609impl Runtime {
610    pub fn new(registry: ExtensionRegistry, config: RuntimeConfig) -> anyhow::Result<Self> {
611        if registry.inference_engines.is_empty() {
612            anyhow::bail!("at least one inference engine must be registered");
613        }
614        validate_runtime_config_reasoning(&config)?;
615        validate_runtime_inference_router_config(&registry, &config)?;
616
617        let bus = EventBus::new(EVENT_BUS_CAPACITY);
618        let thread_store = registry
619            .thread_stores
620            .first()
621            .map(|factory| factory.create());
622        let mut tool_registry = ToolRegistry::default();
623        for contributor in &registry.tools {
624            contributor
625                .contribute(&mut tool_registry)
626                .with_context(|| format!("tool contributor {} failed", contributor.id()))?;
627        }
628        crate::agent_control_tools::contribute_agent_control_tools(&mut tool_registry)?;
629
630        let media_generation = Arc::new(crate::media_generation::MediaGenerationService::new(
631            registry.media_generator_providers.clone(),
632            config.media_generation.clone(),
633        ));
634        tool_registry.replace(Arc::new(
635            crate::media_generation::MediaGenerateImageTool::new(media_generation.clone()),
636        ));
637
638        let team_data_dir = config.team_data_dir.clone();
639        let workspace = config
640            .workspace
641            .clone()
642            .map(PathBuf::from)
643            .unwrap_or(std::env::current_dir()?);
644        let roadmap_data_dir = config
645            .roadmap_data_dir
646            .clone()
647            .unwrap_or_else(|| workspace.join(".roder"));
648        let context_artifacts = thread_store
649            .as_ref()
650            .and_then(|store| store.context_artifact_store())
651            .or_else(|| {
652                thread_store
653                    .as_ref()
654                    .and_then(|store| store.local_thread_root())
655                    .map(FilesystemContextArtifactStore::shared_thread_scoped)
656            })
657            .unwrap_or_else(|| {
658                FilesystemContextArtifactStore::shared_legacy(default_context_artifact_dir())
659            });
660        let goals = Arc::new(RuntimeGoalController::new(
661            bus.clone(),
662            thread_store.clone(),
663        ));
664        let runtime = Self {
665            bus,
666            registry,
667            config: RwLock::new(config),
668            pending_plan_exit: RwLock::new(None),
669            pending_tool_approvals: Mutex::new(HashMap::new()),
670            pending_user_inputs: Mutex::new(HashMap::new()),
671            pending_external_tool_calls: Mutex::new(HashMap::new()),
672            turn_admission: Mutex::new(()),
673            active_turns: RwLock::new(HashMap::new()),
674            turn_drains: RwLock::new(HashMap::new()),
675            provider_turn_cleanups: std::sync::Mutex::new(HashMap::new()),
676            accepting_turns: AtomicBool::new(true),
677            lifecycle_persistence_failures: AtomicUsize::new(0),
678            lifecycle_shutdown_drains: AtomicUsize::new(0),
679            lifecycle_clean_shutdowns: AtomicUsize::new(0),
680            lifecycle_deadline_exceeded: AtomicUsize::new(0),
681            lifecycle_persistence_failed_drains: AtomicUsize::new(0),
682            lifecycle_restart_reconciliations: AtomicUsize::new(0),
683            lifecycle_shutdown_drain_duration_ms_total: AtomicUsize::new(0),
684            provider_cleanup_confirmed: AtomicUsize::new(0),
685            provider_cleanup_timed_out: AtomicUsize::new(0),
686            provider_cleanup_unknown: AtomicUsize::new(0),
687            active_turns_changed: Notify::new(),
688            active_turn_selections: RwLock::new(HashMap::new()),
689            active_turn_contexts: RwLock::new(HashMap::new()),
690            allow_local_workspaces: AtomicBool::new(true),
691            team_member_turn_contexts: Mutex::new(HashMap::new()),
692            workspace: workspace.clone(),
693            teams: TeamManager::new(
694                team_data_dir.unwrap_or_else(crate::teams::default_team_data_dir),
695            ),
696            agent_team_spawn_lock: Mutex::new(()),
697            roadmaps: Mutex::new(roder_roadmap::RoadmapRuntime::new(
698                workspace,
699                roadmap_data_dir,
700            )),
701            goals,
702            context_artifacts,
703            thread_store,
704            thread_item_cache: Mutex::new(ThreadItemCache::default()),
705            tool_registry,
706            media_generation,
707            skills: RwLock::new(SkillRegistry::load(SkillRegistryOptions::new(
708                PathBuf::new(),
709            ))),
710            event_sink_dispatcher: tokio::sync::OnceCell::new(),
711            compaction_hysteresis: crate::compaction_runtime::compaction_hysteresis_state(),
712            agent_swarm_modes: RwLock::new(HashMap::new()),
713            runner_sessions: Arc::new(Mutex::new(HashMap::new())),
714            runner_tool_execution_locks: Mutex::new(HashMap::new()),
715        };
716        runtime.bus.emit(RoderEvent::RuntimeStarted(RuntimeStarted {
717            timestamp: OffsetDateTime::now_utc(),
718        }));
719        for manifest in &runtime.registry.manifests {
720            runtime
721                .bus
722                .emit(RoderEvent::ExtensionRegistered(ExtensionRegistered {
723                    extension_id: manifest.id.clone(),
724                    timestamp: OffsetDateTime::now_utc(),
725                }));
726        }
727        Ok(runtime)
728    }
729
730    pub fn from_engine(engine: Arc<dyn InferenceEngine>) -> anyhow::Result<Self> {
731        let mut builder = roder_api::extension::ExtensionRegistryBuilder::new();
732        builder.inference_engine(engine);
733        Self::new(builder.build()?, RuntimeConfig::default())
734    }
735
736    pub fn fake() -> anyhow::Result<Self> {
737        Self::from_engine(Arc::new(FakeInferenceEngine))
738    }
739
740    pub fn subscribe_events(&self) -> tokio::sync::broadcast::Receiver<EventEnvelope> {
741        self.bus.subscribe()
742    }
743
744    /// Returns process-local, redacted lifecycle health counters. The snapshot
745    /// intentionally has fixed fields and contains no provider, command,
746    /// process, thread, or turn identifiers.
747    pub fn lifecycle_metrics(&self) -> LifecycleMetricsSnapshot {
748        LifecycleMetricsSnapshot {
749            shutdown_drain_count: self.lifecycle_shutdown_drains.load(Ordering::Acquire) as u64,
750            clean_shutdown_count: self.lifecycle_clean_shutdowns.load(Ordering::Acquire) as u64,
751            deadline_exceeded_count: self.lifecycle_deadline_exceeded.load(Ordering::Acquire)
752                as u64,
753            persistence_failed_count: self
754                .lifecycle_persistence_failed_drains
755                .load(Ordering::Acquire) as u64,
756            restart_reconciliation_count: self
757                .lifecycle_restart_reconciliations
758                .load(Ordering::Acquire) as u64,
759            lifecycle_persistence_failure_count: self
760                .lifecycle_persistence_failures
761                .load(Ordering::Acquire) as u64,
762            shutdown_drain_duration_ms_total: self
763                .lifecycle_shutdown_drain_duration_ms_total
764                .load(Ordering::Acquire) as u64,
765            provider_cleanup_confirmed_count: self
766                .provider_cleanup_confirmed
767                .load(Ordering::Acquire) as u64,
768            provider_cleanup_timed_out_count: self
769                .provider_cleanup_timed_out
770                .load(Ordering::Acquire) as u64,
771            provider_cleanup_unknown_count: self.provider_cleanup_unknown.load(Ordering::Acquire)
772                as u64,
773        }
774    }
775
776    async fn persist_turn_lifecycle_record(&self, record: &TurnLifecycleRecord) -> bool {
777        let Some(store) = &self.thread_store else {
778            return true;
779        };
780        let state = match record.extension_state() {
781            Ok(state) => state,
782            Err(_) => {
783                self.lifecycle_persistence_failures
784                    .fetch_add(1, Ordering::AcqRel);
785                return false;
786            }
787        };
788        // Lifecycle diagnostics must not turn a provider result into a failed
789        // turn merely because a third-party store lacks extension-state support.
790        // JSONL/Postgres stores persist this append-only record atomically at
791        // their own storage boundary.
792        if store
793            .append_extension_state(&record.thread_id, &state)
794            .await
795            .is_err()
796        {
797            self.lifecycle_persistence_failures
798                .fetch_add(1, Ordering::AcqRel);
799            return false;
800        }
801        true
802    }
803
804    async fn record_turn_lifecycle(
805        &self,
806        thread_id: ThreadId,
807        turn_id: TurnId,
808        state: TurnLifecycleState,
809        cleanup: TurnCleanupState,
810        reason: Option<TurnLifecycleReason>,
811    ) -> TurnLifecycleRecord {
812        self.record_turn_lifecycle_with_ownership(
813            thread_id,
814            turn_id,
815            state,
816            cleanup,
817            reason,
818            TurnCleanupOwnership::RuntimeTaskOnly,
819        )
820        .await
821    }
822
823    async fn record_turn_lifecycle_with_ownership(
824        &self,
825        thread_id: ThreadId,
826        turn_id: TurnId,
827        state: TurnLifecycleState,
828        cleanup: TurnCleanupState,
829        reason: Option<TurnLifecycleReason>,
830        ownership: TurnCleanupOwnership,
831    ) -> TurnLifecycleRecord {
832        let record = TurnLifecycleRecord::new(
833            thread_id,
834            turn_id,
835            state,
836            cleanup,
837            reason,
838            OffsetDateTime::now_utc(),
839        )
840        .with_ownership(ownership);
841        let _ = self.persist_turn_lifecycle_record(&record).await;
842        self.emit(RoderEvent::TurnLifecycleUpdated(record.clone()))
843            .await;
844        record
845    }
846
847    pub(crate) fn register_provider_turn_cleanup(
848        &self,
849        turn_id: &TurnId,
850        cleanup: Arc<dyn ProviderTurnCleanup>,
851    ) {
852        if let Ok(mut cleanups) = self.provider_turn_cleanups.lock() {
853            cleanups.insert(turn_id.clone(), cleanup);
854        }
855    }
856
857    fn provider_cleanup_ownership(&self, turn_id: &TurnId) -> TurnCleanupOwnership {
858        self.provider_turn_cleanups
859            .lock()
860            .ok()
861            .and_then(|cleanups| cleanups.get(turn_id).cloned())
862            .map(|cleanup| cleanup.ownership())
863            .unwrap_or(TurnCleanupOwnership::RuntimeTaskOnly)
864    }
865
866    async fn await_provider_turn_cleanup(
867        &self,
868        turn_id: &TurnId,
869    ) -> (TurnCleanupState, TurnCleanupOwnership) {
870        let cleanup = self
871            .provider_turn_cleanups
872            .lock()
873            .ok()
874            .and_then(|mut cleanups| cleanups.remove(turn_id));
875        let Some(cleanup) = cleanup else {
876            return (
877                TurnCleanupState::Unknown,
878                TurnCleanupOwnership::RuntimeTaskOnly,
879            );
880        };
881        let ownership = cleanup.ownership();
882        match tokio::time::timeout(PROVIDER_CLEANUP_TIMEOUT, cleanup.wait_for_cleanup()).await {
883            Ok(Ok(())) => {
884                self.provider_cleanup_confirmed
885                    .fetch_add(1, Ordering::AcqRel);
886                (
887                    TurnCleanupState::Completed,
888                    TurnCleanupOwnership::ProviderCleanupConfirmed,
889                )
890            }
891            Ok(Err(_)) => {
892                self.provider_cleanup_unknown.fetch_add(1, Ordering::AcqRel);
893                (TurnCleanupState::Unknown, ownership)
894            }
895            Err(_) => {
896                self.provider_cleanup_timed_out
897                    .fetch_add(1, Ordering::AcqRel);
898                (TurnCleanupState::TimedOut, ownership)
899            }
900        }
901    }
902
903    fn record_lifecycle_drain_outcome(
904        &self,
905        started_at: tokio::time::Instant,
906        outcome: &RuntimeDrainOutcome,
907    ) {
908        self.lifecycle_shutdown_drains
909            .fetch_add(1, Ordering::AcqRel);
910        self.lifecycle_shutdown_drain_duration_ms_total.fetch_add(
911            started_at.elapsed().as_millis().min(usize::MAX as u128) as usize,
912            Ordering::AcqRel,
913        );
914        match outcome {
915            RuntimeDrainOutcome::Clean { .. } => {
916                self.lifecycle_clean_shutdowns
917                    .fetch_add(1, Ordering::AcqRel);
918            }
919            RuntimeDrainOutcome::DeadlineExceeded { .. } => {
920                self.lifecycle_deadline_exceeded
921                    .fetch_add(1, Ordering::AcqRel);
922            }
923            RuntimeDrainOutcome::PersistenceFailed { .. } => {
924                self.lifecycle_persistence_failed_drains
925                    .fetch_add(1, Ordering::AcqRel);
926            }
927        }
928    }
929
930    fn lifecycle_record_for_event(event: &RoderEvent) -> Option<TurnLifecycleRecord> {
931        match event {
932            RoderEvent::TurnStarted(event) => Some(TurnLifecycleRecord::new(
933                event.thread_id.clone(),
934                event.turn_id.clone(),
935                TurnLifecycleState::Running,
936                TurnCleanupState::NotRequested,
937                None,
938                event.timestamp,
939            )),
940            // `run_turn` persists completed turns after it has awaited a
941            // registered provider cleanup handle. Do not let the generic event
942            // projection overwrite that acknowledgement with an unproven state.
943            RoderEvent::TurnCompleted(_) => None,
944            // Some failure paths already have an event before the turn wrapper
945            // reaches its cleanup acknowledgement. Preserve a durable fallback
946            // reason here; the wrapper appends a later record with the more
947            // precise cleanup outcome when a provider registered one.
948            RoderEvent::TurnFailed(event) => Some(TurnLifecycleRecord::new(
949                event.thread_id.clone(),
950                event.turn_id.clone(),
951                TurnLifecycleState::Failed,
952                TurnCleanupState::Unknown,
953                Some(if event.error_kind.as_deref() == Some("deadline_timeout") {
954                    TurnLifecycleReason::DeadlineExceeded
955                } else {
956                    TurnLifecycleReason::ProviderFailure
957                }),
958                event.timestamp,
959            )),
960            RoderEvent::TurnDeadlineExceeded(_) | RoderEvent::TurnInterrupted(_) => None,
961            RoderEvent::TurnLifecycleUpdated(_) => None,
962            _ => None,
963        }
964    }
965
966    pub fn registry(&self) -> &ExtensionRegistry {
967        &self.registry
968    }
969
970    pub fn media_generation(&self) -> Arc<crate::media_generation::MediaGenerationService> {
971        self.media_generation.clone()
972    }
973
974    pub fn context_artifacts(&self) -> roder_api::artifacts::ContextArtifactStore {
975        self.context_artifacts.clone()
976    }
977
978    pub async fn execute_workflow_tool(
979        &self,
980        thread_id: ThreadId,
981        tool_name: &str,
982        arguments: serde_json::Value,
983    ) -> anyhow::Result<ToolResult> {
984        let Some(executor) = self.tool_registry.get(tool_name) else {
985            anyhow::bail!("tool not found: {tool_name}");
986        };
987        let tool_call = ToolCall {
988            id: format!("slash-{tool_name}"),
989            name: tool_name.to_string(),
990            raw_arguments: serde_json::to_string(&arguments)?,
991            arguments,
992            thread_id: thread_id.clone(),
993            turn_id: "slash-command".to_string(),
994        };
995        let runtime_config = self.status().await;
996        let ctx = self.tool_execution_context(
997            thread_id,
998            "slash-command".to_string(),
999            runtime_config.policy_mode,
1000            runtime_config.workspace.as_deref(),
1001            Some(&runtime_config.command_shell),
1002        );
1003        executor.execute(ctx, tool_call).await
1004    }
1005
1006    pub(crate) fn tool_execution_context(
1007        &self,
1008        thread_id: ThreadId,
1009        turn_id: TurnId,
1010        mode: PolicyMode,
1011        workspace: Option<&str>,
1012        command_shell: Option<&str>,
1013    ) -> ToolExecutionContext {
1014        let mut ctx = ToolExecutionContext::new(thread_id, turn_id, mode)
1015            .with_command_shell(command_shell.unwrap_or_default())
1016            .with_process_runner(Arc::new(LocalProcessRunner))
1017            .with_context_artifacts(self.context_artifacts.backend())
1018            .with_goal_controller(self.goals.clone())
1019            .with_subagent_trace_sink(Arc::new(RuntimeSubagentTraceSink::new(
1020                self.bus.clone(),
1021                self.thread_store.clone(),
1022            )))
1023            .with_swarm_progress_sink(Arc::new(RuntimeAgentSwarmProgressSink::new(
1024                self.bus.clone(),
1025                self.thread_store.clone(),
1026            )));
1027        if let Some(workspace) = workspace {
1028            ctx = ctx.with_workspace_handle(Arc::new(ScopedFilesystem::new(workspace)));
1029        }
1030        ctx
1031    }
1032
1033    pub async fn status(&self) -> RuntimeConfig {
1034        self.config.read().await.clone()
1035    }
1036
1037    pub async fn set_skills(&self, skills: SkillRegistry) {
1038        *self.skills.write().await = skills;
1039    }
1040
1041    pub async fn skills_snapshot(&self) -> SkillRegistry {
1042        self.skills.read().await.clone()
1043    }
1044
1045    pub fn workspace(&self) -> PathBuf {
1046        self.workspace.clone()
1047    }
1048
1049    /// Controls whether native workspace tools may execute without an
1050    /// explicit remote-runner binding. Hosted runtimes disable this after
1051    /// construction; ordinary local runtimes retain the compatible default.
1052    pub fn set_allow_local_workspaces(&self, allowed: bool) {
1053        self.allow_local_workspaces
1054            .store(allowed, Ordering::Relaxed);
1055    }
1056
1057    pub fn allows_local_workspaces(&self) -> bool {
1058        self.allow_local_workspaces.load(Ordering::Relaxed)
1059    }
1060
1061    pub async fn set_remote_runner_destination(&self, destination: Option<RunnerDestination>) {
1062        let lifecycle = destination.as_ref().map(|destination| RunnerLifecycle {
1063            destination_id: destination.id.clone(),
1064            provider_id: destination.provider_id.clone(),
1065            state: "configured".to_string(),
1066            session_id: None,
1067            timestamp: OffsetDateTime::now_utc(),
1068        });
1069        self.config.write().await.remote_runner_destination = destination;
1070        if let Some(lifecycle) = lifecycle {
1071            self.emit(RoderEvent::RunnerLifecycle(lifecycle)).await;
1072        } else {
1073            self.emit(RoderEvent::RunnerLifecycle(RunnerLifecycle {
1074                destination_id: "local".to_string(),
1075                provider_id: "local".to_string(),
1076                state: "local_fallback".to_string(),
1077                session_id: None,
1078                timestamp: OffsetDateTime::now_utc(),
1079            }))
1080            .await;
1081        }
1082    }
1083
1084    pub async fn set_file_backed_dynamic_context(&self, enabled: bool) -> RuntimeConfig {
1085        let mut cfg = self.config.write().await;
1086        cfg.file_backed_dynamic_context = enabled;
1087        cfg.clone()
1088    }
1089
1090    pub async fn set_command_shell(&self, shell: String) -> RuntimeConfig {
1091        let mut cfg = self.config.write().await;
1092        cfg.command_shell = shell;
1093        cfg.clone()
1094    }
1095
1096    pub async fn pending_plan_exit(&self) -> Option<PendingPlanExit> {
1097        let mut pending = self.pending_plan_exit.write().await;
1098        let current = pending.clone()?;
1099        if !current.is_expired(OffsetDateTime::now_utc()) {
1100            return Some(current);
1101        }
1102        *pending = None;
1103        drop(pending);
1104        self.emit_plan_exit_resolved(&current, false, self.status().await.policy_mode)
1105            .await;
1106        None
1107    }
1108
1109    pub async fn set_policy_mode(
1110        &self,
1111        mode: PolicyMode,
1112        reason: Option<String>,
1113    ) -> anyhow::Result<RuntimeConfig> {
1114        let mut cfg = self.config.write().await;
1115        let previous_mode = cfg.policy_mode;
1116        cfg.policy_mode = mode;
1117        let next = cfg.clone();
1118        drop(cfg);
1119        self.emit(RoderEvent::PolicyModeChanged(PolicyModeChanged {
1120            thread_id: "runtime".to_string(),
1121            turn_id: None,
1122            previous_mode,
1123            new_mode: mode,
1124            reason,
1125            timestamp: OffsetDateTime::now_utc(),
1126        }))
1127        .await;
1128        self.auto_resolve_pending_tool_approvals_for_mode(mode)
1129            .await;
1130        Ok(next)
1131    }
1132
1133    /// Toggle agent-swarm mode for the runtime (roadmap 104). When enabled, the
1134    /// swarm reminder is injected into each turn's developer instructions so
1135    /// every app-server/SDK client gets it, not only the TUI.
1136    pub async fn set_agent_swarm_mode(
1137        &self,
1138        enabled: bool,
1139        trigger: roder_api::subagents::AgentSwarmModeTrigger,
1140    ) -> anyhow::Result<RuntimeConfig> {
1141        let mut cfg = self.config.write().await;
1142        cfg.agent_swarm_mode = enabled;
1143        let next = cfg.clone();
1144        drop(cfg);
1145        self.emit(RoderEvent::AgentSwarmModeChanged(
1146            roder_api::subagents::AgentSwarmModeChanged {
1147                thread_id: "runtime".to_string(),
1148                turn_id: None,
1149                enabled,
1150                trigger,
1151                timestamp: OffsetDateTime::now_utc(),
1152            },
1153        ))
1154        .await;
1155        Ok(next)
1156    }
1157
1158    /// Toggle agent-swarm mode for a single thread (roadmap 104). The override is
1159    /// stored per thread so toggling swarm mode on one thread does not leak the
1160    /// reminder into other threads sharing the runtime; absent threads fall back
1161    /// to the runtime-global default. The emitted `AgentSwarmModeChanged` event
1162    /// carries the real `thread_id`.
1163    pub async fn set_agent_swarm_mode_for_thread(
1164        &self,
1165        thread_id: &str,
1166        enabled: bool,
1167        trigger: roder_api::subagents::AgentSwarmModeTrigger,
1168    ) -> bool {
1169        {
1170            let mut modes = self.agent_swarm_modes.write().await;
1171            modes.insert(thread_id.to_string(), enabled);
1172        }
1173        self.emit(RoderEvent::AgentSwarmModeChanged(
1174            roder_api::subagents::AgentSwarmModeChanged {
1175                thread_id: thread_id.to_string(),
1176                turn_id: None,
1177                enabled,
1178                trigger,
1179                timestamp: OffsetDateTime::now_utc(),
1180            },
1181        ))
1182        .await;
1183        enabled
1184    }
1185
1186    /// Resolve whether agent-swarm mode is active for `thread_id`: the per-thread
1187    /// override when present, otherwise the runtime-global default. The turn loop
1188    /// uses this to decide whether to inject the swarm reminder.
1189    pub async fn effective_agent_swarm_mode_for_thread(&self, thread_id: &str) -> bool {
1190        if let Some(enabled) = self.agent_swarm_modes.read().await.get(thread_id).copied() {
1191            return enabled;
1192        }
1193        self.status().await.agent_swarm_mode
1194    }
1195
1196    pub async fn set_hosted_web_search(
1197        &self,
1198        mode: HostedWebSearchMode,
1199    ) -> anyhow::Result<RuntimeConfig> {
1200        let mut cfg = self.config.write().await;
1201        cfg.hosted_web_search = HostedWebSearchConfig { mode };
1202        Ok(cfg.clone())
1203    }
1204
1205    async fn auto_resolve_pending_tool_approvals_for_mode(&self, mode: PolicyMode) {
1206        let gate = DefaultPolicyGate::new();
1207        let mut pending = self.pending_tool_approvals.lock().await;
1208        let approval_ids = pending
1209            .iter()
1210            .filter_map(|(approval_id, approval)| {
1211                let ctx = ToolExecutionContext::new(
1212                    approval.thread_id.clone(),
1213                    approval.turn_id.clone(),
1214                    mode,
1215                );
1216                matches!(
1217                    gate.decide(&approval.call, mode, &ctx),
1218                    PolicyDecision::AutoApproved { .. }
1219                )
1220                .then_some(approval_id.clone())
1221            })
1222            .collect::<Vec<_>>();
1223        let approvals = approval_ids
1224            .into_iter()
1225            .filter_map(|approval_id| {
1226                pending
1227                    .remove(&approval_id)
1228                    .map(|approval| (approval_id, approval))
1229            })
1230            .collect::<Vec<_>>();
1231        drop(pending);
1232
1233        for (approval_id, approval) in approvals {
1234            let ctx = ToolExecutionContext::new(
1235                approval.thread_id.clone(),
1236                approval.turn_id.clone(),
1237                mode,
1238            );
1239            let decision = gate.decide(&approval.call, mode, &ctx);
1240            self.emit(RoderEvent::PolicyDecisionRecorded(PolicyDecisionRecorded {
1241                thread_id: approval.thread_id.clone(),
1242                turn_id: approval.turn_id.clone(),
1243                tool_id: approval.tool_id.clone(),
1244                tool_name: approval.tool_name.clone(),
1245                mode,
1246                decision,
1247                timestamp: OffsetDateTime::now_utc(),
1248            }))
1249            .await;
1250            if mode == PolicyMode::Bypass {
1251                self.emit(RoderEvent::PolicyBypassActive(PolicyBypassActive {
1252                    thread_id: approval.thread_id.clone(),
1253                    turn_id: approval.turn_id.clone(),
1254                    tool_id: approval.tool_id.clone(),
1255                    tool_name: approval.tool_name.clone(),
1256                    timestamp: OffsetDateTime::now_utc(),
1257                }))
1258                .await;
1259            }
1260            self.emit(RoderEvent::ApprovalResolved(ApprovalResolved {
1261                thread_id: approval.thread_id,
1262                turn_id: approval.turn_id,
1263                approval_id,
1264                tool_id: approval.tool_id,
1265                tool_name: approval.tool_name,
1266                approved: true,
1267                timestamp: OffsetDateTime::now_utc(),
1268            }))
1269            .await;
1270            let _ = approval.tx.send(true);
1271        }
1272    }
1273
1274    pub async fn record_pending_plan_exit(&self, pending: PendingPlanExit) {
1275        *self.pending_plan_exit.write().await = Some(pending.clone());
1276        self.emit(RoderEvent::PolicyExitPlanRequested(
1277            PolicyExitPlanRequested {
1278                thread_id: pending.thread_id,
1279                turn_id: pending.turn_id,
1280                request_id: pending.request_id,
1281                target_mode: pending.target_mode,
1282                plan_summary: pending.plan_summary,
1283                next_steps: pending.next_steps,
1284                timestamp: OffsetDateTime::now_utc(),
1285            },
1286        ))
1287        .await;
1288    }
1289
1290    pub async fn resolve_pending_plan_exit(
1291        &self,
1292        request_id: &str,
1293        approved: bool,
1294    ) -> anyhow::Result<Option<PendingPlanExit>> {
1295        let mut pending = self.pending_plan_exit.write().await;
1296        let Some(current) = pending.clone() else {
1297            return Ok(None);
1298        };
1299        if current.request_id != request_id {
1300            anyhow::bail!("pending plan exit request {request_id:?} was not found");
1301        }
1302        *pending = None;
1303        drop(pending);
1304
1305        let approved = approved && !current.is_expired(OffsetDateTime::now_utc());
1306        let resolved_mode = if approved {
1307            let mut cfg = self.config.write().await;
1308            let previous_mode = cfg.policy_mode;
1309            cfg.policy_mode = current.target_mode;
1310            drop(cfg);
1311            self.emit(RoderEvent::PolicyModeChanged(PolicyModeChanged {
1312                thread_id: current.thread_id.clone(),
1313                turn_id: Some(current.turn_id.clone()),
1314                previous_mode,
1315                new_mode: current.target_mode,
1316                reason: Some("approved plan exit".to_string()),
1317                timestamp: OffsetDateTime::now_utc(),
1318            }))
1319            .await;
1320            self.auto_resolve_pending_tool_approvals_for_mode(current.target_mode)
1321                .await;
1322            current.target_mode
1323        } else {
1324            self.status().await.policy_mode
1325        };
1326        self.emit_plan_exit_resolved(&current, approved, resolved_mode)
1327            .await;
1328        Ok(Some(current))
1329    }
1330
1331    pub async fn resolve_tool_approval(
1332        &self,
1333        approval_id: &str,
1334        approved: bool,
1335    ) -> anyhow::Result<bool> {
1336        let pending = self.pending_tool_approvals.lock().await.remove(approval_id);
1337        let Some(pending) = pending else {
1338            return Ok(false);
1339        };
1340        self.emit(RoderEvent::ApprovalResolved(ApprovalResolved {
1341            thread_id: pending.thread_id,
1342            turn_id: pending.turn_id,
1343            approval_id: approval_id.to_string(),
1344            tool_id: pending.tool_id,
1345            tool_name: pending.tool_name,
1346            approved,
1347            timestamp: OffsetDateTime::now_utc(),
1348        }))
1349        .await;
1350        let _ = pending.tx.send(approved);
1351        Ok(true)
1352    }
1353
1354    pub async fn request_app_server_tool_approval(
1355        &self,
1356        call: ToolCall,
1357        reason: Option<String>,
1358    ) -> anyhow::Result<bool> {
1359        let approval_id = call.id.clone();
1360        let (tx, rx) = oneshot::channel();
1361        self.pending_tool_approvals.lock().await.insert(
1362            approval_id.clone(),
1363            PendingToolApproval {
1364                thread_id: call.thread_id.clone(),
1365                turn_id: call.turn_id.clone(),
1366                tool_id: call.id.clone(),
1367                tool_name: call.name.clone(),
1368                call: call.clone(),
1369                tx,
1370            },
1371        );
1372        self.emit(RoderEvent::ApprovalRequested(ApprovalRequested {
1373            thread_id: call.thread_id.clone(),
1374            turn_id: call.turn_id.clone(),
1375            approval_id,
1376            tool_id: call.id.clone(),
1377            tool_name: call.name.clone(),
1378            reason,
1379            timestamp: OffsetDateTime::now_utc(),
1380        }))
1381        .await;
1382        Ok(rx.await.unwrap_or(false))
1383    }
1384
1385    /// Completes a pending host-executed tool call (`tools/resolve`). Returns false when the
1386    /// request id is unknown, already resolved, timed out, or cancelled by a turn interrupt.
1387    pub async fn resolve_external_tool_call(
1388        &self,
1389        request_id: &str,
1390        resolution: ExternalToolResolution,
1391    ) -> anyhow::Result<bool> {
1392        let pending = self
1393            .pending_external_tool_calls
1394            .lock()
1395            .await
1396            .remove(request_id);
1397        let Some(pending) = pending else {
1398            return Ok(false);
1399        };
1400        self.emit(RoderEvent::ExternalToolCallResolved(
1401            ExternalToolCallResolved {
1402                thread_id: pending.thread_id,
1403                turn_id: pending.turn_id,
1404                request_id: request_id.to_string(),
1405                tool_id: pending.tool_id,
1406                tool_name: pending.tool_name,
1407                outcome: ExternalToolCallOutcome::Resolved,
1408                is_error: resolution.is_error,
1409                timestamp: OffsetDateTime::now_utc(),
1410            },
1411        ))
1412        .await;
1413        let _ = pending.tx.send(resolution);
1414        Ok(true)
1415    }
1416
1417    /// Drops every pending external tool call for the turn and reports them as cancelled so an
1418    /// interrupt does not leak requests waiting on `tools/resolve`.
1419    async fn cancel_pending_external_tool_calls_for_turn(&self, turn_id: &TurnId) {
1420        let cancelled = {
1421            let mut pending = self.pending_external_tool_calls.lock().await;
1422            let request_ids = pending
1423                .iter()
1424                .filter(|(_, call)| &call.turn_id == turn_id)
1425                .map(|(request_id, _)| request_id.clone())
1426                .collect::<Vec<_>>();
1427            request_ids
1428                .into_iter()
1429                .filter_map(|request_id| pending.remove(&request_id).map(|call| (request_id, call)))
1430                .collect::<Vec<_>>()
1431        };
1432        for (request_id, call) in cancelled {
1433            self.emit(RoderEvent::ExternalToolCallResolved(
1434                ExternalToolCallResolved {
1435                    thread_id: call.thread_id,
1436                    turn_id: call.turn_id,
1437                    request_id,
1438                    tool_id: call.tool_id,
1439                    tool_name: call.tool_name,
1440                    outcome: ExternalToolCallOutcome::Cancelled,
1441                    is_error: true,
1442                    timestamp: OffsetDateTime::now_utc(),
1443                },
1444            ))
1445            .await;
1446        }
1447    }
1448
1449    pub async fn resolve_user_input(
1450        &self,
1451        request_id: &str,
1452        answers: serde_json::Value,
1453    ) -> anyhow::Result<bool> {
1454        let pending = self.pending_user_inputs.lock().await.remove(request_id);
1455        let Some(pending) = pending else {
1456            return Ok(false);
1457        };
1458        self.emit(RoderEvent::UserInputResolved(UserInputResolved {
1459            thread_id: pending.thread_id,
1460            turn_id: pending.turn_id,
1461            request_id: request_id.to_string(),
1462            answers: answers.clone(),
1463            timestamp: OffsetDateTime::now_utc(),
1464        }))
1465        .await;
1466        let _ = pending.tx.send(answers);
1467        Ok(true)
1468    }
1469
1470    async fn emit_plan_exit_resolved(
1471        &self,
1472        current: &PendingPlanExit,
1473        approved: bool,
1474        resolved_mode: PolicyMode,
1475    ) {
1476        self.emit(RoderEvent::PolicyExitPlanResolved(PolicyExitPlanResolved {
1477            thread_id: current.thread_id.clone(),
1478            turn_id: current.turn_id.clone(),
1479            request_id: current.request_id.clone(),
1480            approved,
1481            target_mode: current.target_mode,
1482            resolved_mode,
1483            timestamp: OffsetDateTime::now_utc(),
1484        }))
1485        .await;
1486    }
1487
1488    pub async fn select_provider(
1489        &self,
1490        provider: String,
1491        model: Option<String>,
1492        reasoning: Option<String>,
1493    ) -> anyhow::Result<RuntimeConfig> {
1494        let next = self
1495            .preview_provider_selection(provider, model, reasoning)
1496            .await?;
1497        let mut cfg = self.config.write().await;
1498        *cfg = next;
1499        Ok(cfg.clone())
1500    }
1501
1502    pub async fn preview_provider_selection(
1503        &self,
1504        provider: String,
1505        model: Option<String>,
1506        reasoning: Option<String>,
1507    ) -> anyhow::Result<RuntimeConfig> {
1508        self.engine_for(&provider)?;
1509        let mut cfg = self.config.read().await.clone();
1510        cfg.default_provider = provider;
1511        if let Some(model) = model {
1512            cfg.default_model = model;
1513        }
1514        if let Some(reasoning) = reasoning {
1515            if reasoning == REASONING_NONE
1516                && !model_supports_reasoning(&cfg.default_model, &reasoning)
1517            {
1518                return Ok(cfg.clone());
1519            }
1520            validate_reasoning_effort(&cfg.default_model, &reasoning)?;
1521            cfg.reasoning = Some(reasoning);
1522        }
1523        Ok(cfg)
1524    }
1525
1526    pub async fn effective_reasoning(&self) -> String {
1527        let cfg = self.config.read().await;
1528        effective_reasoning_for_model(&cfg, &cfg.default_model)
1529    }
1530
1531    pub fn effective_reasoning_for_config(cfg: &RuntimeConfig) -> String {
1532        effective_reasoning_for_model(cfg, &cfg.default_model)
1533    }
1534
1535    pub async fn set_dynamic_workflow_effort(
1536        &self,
1537        effort_profile: DynamicWorkflowEffortProfile,
1538    ) -> RuntimeConfig {
1539        let mut cfg = self.config.write().await;
1540        cfg.dynamic_workflows.effort_profile = effort_profile;
1541        cfg.clone()
1542    }
1543
1544    pub async fn dynamic_workflow_trigger_decision(
1545        &self,
1546        message: &str,
1547    ) -> WorkflowTriggerDecision {
1548        let cfg = self.config.read().await;
1549        classify_workflow_trigger(message, &cfg.dynamic_workflows)
1550    }
1551
1552    pub async fn create_thread(&self, title: Option<String>) -> anyhow::Result<ThreadMetadata> {
1553        self.create_thread_with(CreateThreadRequest {
1554            title,
1555            workspace: self.workspace.display().to_string(),
1556            workspace_id: None,
1557            root_id: None,
1558            provider: None,
1559            model: None,
1560            selection_mode: None,
1561            tool_allowlist: Vec::new(),
1562            developer_instructions: None,
1563            external_tools: Vec::new(),
1564            runner: None,
1565        })
1566        .await
1567    }
1568
1569    /**
1570     * Resolves an explicit thread-runner selection into a persisted binding.
1571     * Fails fast at thread creation when the provider is missing or rejects
1572     * the destination, instead of surfacing the error on the first tool call.
1573     */
1574    async fn resolve_thread_runner_binding(
1575        &self,
1576        thread_id: &str,
1577        selection: ThreadRunnerSelection,
1578    ) -> anyhow::Result<ThreadRunnerBinding> {
1579        let provider = self
1580            .registry
1581            .remote_runner_providers
1582            .iter()
1583            .find(|provider| provider.id() == selection.provider_id)
1584            .cloned()
1585            .ok_or_else(|| {
1586                anyhow::anyhow!(
1587                    "remote runner provider {:?} is not installed",
1588                    selection.provider_id
1589                )
1590            })?;
1591        let workspace = selection.workspace.trim();
1592        anyhow::ensure!(
1593            std::path::Path::new(workspace).is_absolute(),
1594            "runner workspace must be an absolute path on the runner: {workspace:?}"
1595        );
1596        let mut read_roots = Vec::with_capacity(selection.read_roots.len());
1597        for read_root in &selection.read_roots {
1598            let trimmed = read_root.trim();
1599            anyhow::ensure!(
1600                std::path::Path::new(trimmed).is_absolute(),
1601                "runner read root must be an absolute path on the runner: {trimmed:?}"
1602            );
1603            read_roots.push(PathBuf::from(trimmed));
1604        }
1605        let destination = RunnerDestination {
1606            id: format!("thread-{thread_id}"),
1607            provider_id: selection.provider_id,
1608            config: selection.config,
1609            default_manifest: roder_api::remote_runner::RunnerManifest::default(),
1610        };
1611        provider.validate_destination(&destination).await?;
1612        Ok(ThreadRunnerBinding {
1613            destination,
1614            workspace: PathBuf::from(workspace),
1615            read_roots,
1616        })
1617    }
1618
1619    /**
1620     * Build a per-thread binding from a runtime-level destination (selected via
1621     * the TUI runner picker or config `default_destination`) when the
1622     * destination's provider advertises a default workspace. Returns `None` when
1623     * there is no destination or the provider opts out (no default workspace),
1624     * preserving the legacy behavior where such threads keep local tools.
1625     */
1626    async fn synthesize_runtime_runner_binding(
1627        &self,
1628        thread_id: &str,
1629        destination: Option<RunnerDestination>,
1630    ) -> anyhow::Result<Option<ThreadRunnerBinding>> {
1631        let Some(destination) = destination else {
1632            return Ok(None);
1633        };
1634        let Some(provider) = self
1635            .registry
1636            .remote_runner_providers
1637            .iter()
1638            .find(|provider| provider.id() == destination.provider_id)
1639        else {
1640            return Ok(None);
1641        };
1642        // An explicit workspace in the destination config wins over the
1643        // provider default; absence of both means the provider opts out.
1644        let workspace = destination
1645            .config
1646            .get("working_dir")
1647            .and_then(serde_json::Value::as_str)
1648            .map(str::to_string)
1649            .or_else(|| provider.default_workspace());
1650        let Some(workspace) = workspace else {
1651            return Ok(None);
1652        };
1653        let selection = ThreadRunnerSelection {
1654            provider_id: destination.provider_id.clone(),
1655            config: destination.config.clone(),
1656            workspace,
1657            read_roots: Vec::new(),
1658        };
1659        Ok(Some(
1660            self.resolve_thread_runner_binding(thread_id, selection)
1661                .await?,
1662        ))
1663    }
1664
1665    /// Validates a runner selection without creating a thread; the placeholder destination id only appears in error messages.
1666    pub async fn validate_thread_runner_selection(
1667        &self,
1668        selection: ThreadRunnerSelection,
1669    ) -> anyhow::Result<()> {
1670        self.resolve_thread_runner_binding("validate", selection)
1671            .await
1672            .map(|_| ())
1673    }
1674
1675    pub async fn create_thread_with(
1676        &self,
1677        req: CreateThreadRequest,
1678    ) -> anyhow::Result<ThreadMetadata> {
1679        let cfg = self.config.read().await.clone();
1680        let now = OffsetDateTime::now_utc();
1681        let workspace = validate_thread_workspace(&req.workspace)?;
1682        let provider = req.provider.unwrap_or(cfg.default_provider);
1683        let model = req.model.unwrap_or(cfg.default_model);
1684        let selection_mode = req
1685            .selection_mode
1686            .unwrap_or_else(|| ModelSelectionMode::manual(provider.clone(), model.clone(), None));
1687        let thread_id = uuid::Uuid::new_v4().to_string();
1688        let runner_binding = match req.runner {
1689            Some(selection) => Some(
1690                self.resolve_thread_runner_binding(&thread_id, selection)
1691                    .await?,
1692            ),
1693            // No explicit per-thread selection: if a runtime-level destination
1694            // is active and its provider advertises a default workspace, bind
1695            // the new thread so its coding tools route into the runner. This is
1696            // what makes a TUI/config-selected runner (e.g. Blaxel) actually
1697            // execute tools in the sandbox instead of locally. Providers that
1698            // return no default workspace stay unbound (legacy local-tools).
1699            None => {
1700                self.synthesize_runtime_runner_binding(
1701                    &thread_id,
1702                    cfg.remote_runner_destination.clone(),
1703                )
1704                .await?
1705            }
1706        };
1707        let runner_destination = runner_binding
1708            .as_ref()
1709            .map(|binding| binding.destination.clone())
1710            .or_else(|| cfg.remote_runner_destination.clone());
1711        let metadata = ThreadMetadata {
1712            thread_id,
1713            title: req.title,
1714            workspace,
1715            workspace_id: req.workspace_id,
1716            root_id: req.root_id,
1717            provider: Some(provider),
1718            model: Some(model),
1719            selection_mode: Some(selection_mode),
1720            tool_allowlist: req.tool_allowlist,
1721            developer_instructions: req.developer_instructions,
1722            external_tools: req.external_tools,
1723            runner_destination,
1724            runner_state: None,
1725            runner_binding,
1726            created_at: now,
1727            updated_at: now,
1728            message_count: 0,
1729            usage: None,
1730            parent_thread_id: None,
1731            forked_from_turn_id: None,
1732            workspace_fork: None,
1733        };
1734
1735        let metadata = if let Some(store) = &self.thread_store {
1736            store.create_thread(metadata).await?
1737        } else {
1738            metadata
1739        };
1740        self.emit(RoderEvent::ThreadCreated(ThreadCreated {
1741            thread_id: metadata.thread_id.clone(),
1742            timestamp: OffsetDateTime::now_utc(),
1743        }))
1744        .await;
1745        Ok(metadata)
1746    }
1747
1748    pub async fn list_threads(&self) -> anyhow::Result<Vec<ThreadMetadata>> {
1749        if let Some(store) = &self.thread_store {
1750            return store.list_threads().await;
1751        }
1752        Ok(Vec::new())
1753    }
1754
1755    pub async fn list_threads_page(
1756        &self,
1757        options: roder_api::thread::ThreadListOptions,
1758    ) -> anyhow::Result<roder_api::thread::ThreadListPage> {
1759        if let Some(store) = &self.thread_store {
1760            return store.list_threads_page(options).await;
1761        }
1762        Ok(roder_api::thread::ThreadListPage::default())
1763    }
1764
1765    pub async fn load_thread_metadata(
1766        &self,
1767        thread_id: &str,
1768    ) -> anyhow::Result<Option<roder_api::thread::ThreadMetadata>> {
1769        if let Some(store) = &self.thread_store {
1770            return store.load_thread_metadata(&thread_id.to_string()).await;
1771        }
1772        Ok(None)
1773    }
1774
1775    pub async fn archive_thread(&self, thread_id: &str) -> anyhow::Result<bool> {
1776        let archived = if let Some(store) = &self.thread_store {
1777            store.archive_thread(&thread_id.to_string()).await?
1778        } else {
1779            false
1780        };
1781        if archived {
1782            self.thread_item_cache
1783                .lock()
1784                .await
1785                .remove_thread(&thread_id.to_string());
1786            self.evict_runner_session(&thread_id.to_string()).await;
1787        }
1788        Ok(archived)
1789    }
1790
1791    pub async fn start_team(&self, req: TeamStartRequest) -> anyhow::Result<TeamState> {
1792        let cfg = self.config.read().await.clone();
1793        let workspace = self.workspace.display().to_string();
1794        let lead_thread_id = match req.lead_thread_id {
1795            Some(thread_id) => thread_id,
1796            None => {
1797                self.create_thread_with(CreateThreadRequest {
1798                    title: Some("Team lead".to_string()),
1799                    workspace: workspace.clone(),
1800                    workspace_id: None,
1801                    root_id: None,
1802                    provider: None,
1803                    model: None,
1804                    selection_mode: None,
1805                    tool_allowlist: Vec::new(),
1806                    developer_instructions: None,
1807                    external_tools: Vec::new(),
1808                    runner: None,
1809                })
1810                .await?
1811                .thread_id
1812            }
1813        };
1814        let team_id = uuid::Uuid::new_v4().to_string();
1815        let active_lead_turn_id = self.active_turn_for_thread(&lead_thread_id).await;
1816        let lead_selection = match active_lead_turn_id.as_ref() {
1817            Some(turn_id) => self
1818                .active_turn_selections
1819                .read()
1820                .await
1821                .get(turn_id)
1822                .cloned(),
1823            None => self.selection_mode_for_thread(&lead_thread_id).await?,
1824        };
1825        let lead_concrete_selection = lead_selection
1826            .as_ref()
1827            .map(ModelSelectionMode::concrete_selection);
1828        let mut lead = crate::teams::lead_member(
1829            lead_thread_id.clone(),
1830            lead_concrete_selection
1831                .as_ref()
1832                .map(|selection| selection.provider.clone())
1833                .or_else(|| Some(cfg.default_provider.clone())),
1834            lead_concrete_selection
1835                .as_ref()
1836                .map(|selection| selection.model.clone())
1837                .or_else(|| Some(cfg.default_model.clone())),
1838            cfg.policy_mode,
1839        );
1840        if let Some(turn_id) = active_lead_turn_id {
1841            lead.current_turn_id = Some(turn_id);
1842            lead.status = TeamMemberStatus::Running;
1843        }
1844        let mut members = vec![lead];
1845
1846        for (index, member) in req.members.into_iter().enumerate() {
1847            let thread = self
1848                .create_thread_with(CreateThreadRequest {
1849                    title: Some(member.name.clone()),
1850                    workspace: workspace.clone(),
1851                    workspace_id: None,
1852                    root_id: None,
1853                    provider: member.model_provider.clone(),
1854                    model: member.model.clone(),
1855                    selection_mode: None,
1856                    tool_allowlist: Vec::new(),
1857                    developer_instructions: None,
1858                    external_tools: Vec::new(),
1859                    runner: None,
1860                })
1861                .await?;
1862            let member_id = format!("member-{}", index + 1);
1863            let descriptor = crate::teams::teammate_member(
1864                member_id.clone(),
1865                member.name,
1866                thread.thread_id.clone(),
1867                member.model_provider.or(thread.provider),
1868                member.model.or(thread.model),
1869                cfg.policy_mode,
1870            );
1871            members.push(descriptor);
1872        }
1873
1874        let now = OffsetDateTime::now_utc();
1875        let team = self
1876            .teams
1877            .insert(TeamState {
1878                id: team_id.clone(),
1879                lead_thread_id: lead_thread_id.clone(),
1880                display_mode: req.display_mode,
1881                members,
1882                mailbox: Vec::new(),
1883                tasks: Vec::new(),
1884                created_at: now,
1885                updated_at: now,
1886            })
1887            .await?;
1888        self.emit(RoderEvent::TeamStarted(TeamStarted {
1889            team_id: team_id.clone(),
1890            lead_thread_id,
1891            display_mode: team.display_mode,
1892            members: team.members.clone(),
1893            tasks: team.tasks.clone(),
1894            timestamp: OffsetDateTime::now_utc(),
1895        }))
1896        .await;
1897        for member in team
1898            .members
1899            .iter()
1900            .filter(|member| member.role != roder_api::teams::TeamMemberRole::Lead)
1901        {
1902            self.emit(RoderEvent::TeamMemberStarted(TeamMemberStarted {
1903                team_id: team_id.clone(),
1904                member: member.clone(),
1905                timestamp: OffsetDateTime::now_utc(),
1906            }))
1907            .await;
1908        }
1909        Ok(team)
1910    }
1911
1912    pub async fn list_teams(&self) -> Vec<TeamState> {
1913        let active_thread_ids = self
1914            .active_turns
1915            .read()
1916            .await
1917            .values()
1918            .map(|handle| handle.thread_id.clone())
1919            .collect::<std::collections::HashSet<_>>();
1920        self.teams
1921            .list()
1922            .await
1923            .into_iter()
1924            .map(|team| runtime_local_team_view(team, &active_thread_ids))
1925            .collect()
1926    }
1927
1928    pub async fn read_team(&self, team_id: &str) -> Option<TeamState> {
1929        let active_thread_ids = self
1930            .active_turns
1931            .read()
1932            .await
1933            .values()
1934            .map(|handle| handle.thread_id.clone())
1935            .collect::<std::collections::HashSet<_>>();
1936        self.teams
1937            .get(team_id)
1938            .await
1939            .map(|team| runtime_local_team_view(team, &active_thread_ids))
1940    }
1941
1942    pub async fn start_team_member(
1943        &self,
1944        team_id: &str,
1945        req: TeamMemberStartRequest,
1946    ) -> anyhow::Result<TeamState> {
1947        self.start_team_member_with_selection(team_id, req, None)
1948            .await
1949    }
1950
1951    pub async fn message_team_member(
1952        self: &Arc<Self>,
1953        team_id: &str,
1954        member_id: &str,
1955        message: String,
1956    ) -> anyhow::Result<TurnId> {
1957        let team = self
1958            .read_team(team_id)
1959            .await
1960            .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
1961        self.followup_team_member(&team.lead_thread_id, team_id, member_id, message)
1962            .await
1963    }
1964
1965    /// Queue a mailbox message without starting an idle agent. If the target is
1966    /// already running, the message is delivered at the next inference boundary.
1967    pub(crate) async fn queue_team_member_message(
1968        self: &Arc<Self>,
1969        caller_thread_id: &ThreadId,
1970        team_id: &str,
1971        member_id: &str,
1972        message: String,
1973    ) -> anyhow::Result<Option<TurnId>> {
1974        let _delivery_guard = self.agent_team_spawn_lock.lock().await;
1975        let team = self
1976            .read_team(team_id)
1977            .await
1978            .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
1979        let member = team
1980            .members
1981            .iter()
1982            .find(|member| member.id == member_id)
1983            .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
1984            .clone();
1985        if member.status == TeamMemberStatus::Closed {
1986            anyhow::bail!("subagent {} is closed", member.name);
1987        }
1988        let from_member_id = team
1989            .members
1990            .iter()
1991            .find(|candidate| candidate.thread_id == *caller_thread_id)
1992            .map(|candidate| candidate.id.clone());
1993        self.teams
1994            .append_mailbox_message(
1995                team_id,
1996                from_member_id,
1997                member_id.to_string(),
1998                TeamMailboxMessageKind::Message,
1999                message,
2000            )
2001            .await?;
2002        let Some(turn_id) = self.active_turn_for_thread(&member.thread_id).await else {
2003            return Ok(None);
2004        };
2005        self.deliver_pending_team_mailbox(team_id, member_id, member.thread_id, turn_id.clone())
2006            .await?;
2007        Ok(Some(turn_id))
2008    }
2009
2010    /// Deliver queued messages and start an idle agent, or steer its active turn.
2011    pub(crate) async fn followup_team_member(
2012        self: &Arc<Self>,
2013        caller_thread_id: &ThreadId,
2014        team_id: &str,
2015        member_id: &str,
2016        message: String,
2017    ) -> anyhow::Result<TurnId> {
2018        let _spawn_guard = self.agent_team_spawn_lock.lock().await;
2019        self.followup_team_member_locked(caller_thread_id, team_id, member_id, message)
2020            .await
2021    }
2022
2023    async fn followup_team_member_locked(
2024        self: &Arc<Self>,
2025        caller_thread_id: &ThreadId,
2026        team_id: &str,
2027        member_id: &str,
2028        message: String,
2029    ) -> anyhow::Result<TurnId> {
2030        let team = self
2031            .read_team(team_id)
2032            .await
2033            .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2034        let member = team
2035            .members
2036            .iter()
2037            .find(|member| member.id == member_id)
2038            .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2039            .clone();
2040        if member.status == TeamMemberStatus::Closed {
2041            anyhow::bail!("subagent {} is closed", member.name);
2042        }
2043        let from_member_id = team
2044            .members
2045            .iter()
2046            .find(|candidate| candidate.thread_id == *caller_thread_id)
2047            .map(|candidate| candidate.id.clone());
2048        self.teams
2049            .append_mailbox_message(
2050                team_id,
2051                from_member_id,
2052                member_id.to_string(),
2053                TeamMailboxMessageKind::NewTask,
2054                message,
2055            )
2056            .await?;
2057        if let Some(turn_id) = self.active_turn_for_thread(&member.thread_id).await {
2058            self.deliver_pending_team_mailbox(
2059                team_id,
2060                member_id,
2061                member.thread_id,
2062                turn_id.clone(),
2063            )
2064            .await?;
2065            return Ok(turn_id);
2066        }
2067
2068        self.ensure_codex_v2_team_capacity(&team, &member.id, None)
2069            .await?;
2070        let metadata = self.load_thread_metadata(&member.thread_id).await?;
2071        let inherited_context = self
2072            .team_member_turn_contexts
2073            .lock()
2074            .await
2075            .get(&member.thread_id)
2076            .cloned();
2077        let workspace = inherited_context
2078            .as_ref()
2079            .map(|context| context.workspace.clone())
2080            .or_else(|| metadata.as_ref().map(|metadata| metadata.workspace.clone()))
2081            .unwrap_or_else(|| self.workspace.display().to_string());
2082        let member_thread_id = member.thread_id;
2083        let turn_id = self
2084            .start_turn(StartTurnRequest {
2085                thread_id: member_thread_id.clone(),
2086                message: String::new(),
2087                images: Vec::new(),
2088                provider_override: member.model_provider,
2089                model_override: member.model,
2090                reasoning_override: metadata
2091                    .as_ref()
2092                    .and_then(|metadata| metadata.selection_mode.as_ref())
2093                    .and_then(ModelSelectionMode::reasoning)
2094                    .map(str::to_string),
2095                workspace,
2096                instructions: inherited_context
2097                    .as_ref()
2098                    .map(|context| context.instructions.clone())
2099                    .unwrap_or_else(crate::default_instructions),
2100                developer_context: inherited_context
2101                    .as_ref()
2102                    .and_then(|context| context.developer_context.clone()),
2103                task_ledger_required: false,
2104            })
2105            .await?;
2106        Ok(turn_id)
2107    }
2108
2109    pub async fn set_team_member_policy_mode(
2110        &self,
2111        team_id: &str,
2112        member_id: &str,
2113        policy_mode: PolicyMode,
2114    ) -> anyhow::Result<TeamState> {
2115        self.teams
2116            .set_member_policy_mode(team_id, member_id, policy_mode)
2117            .await
2118    }
2119
2120    pub async fn interrupt_team_member(
2121        &self,
2122        team_id: &str,
2123        member_id: &str,
2124    ) -> anyhow::Result<Option<TurnId>> {
2125        let _interrupt_guard = self.agent_team_spawn_lock.lock().await;
2126        let team = self
2127            .read_team(team_id)
2128            .await
2129            .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2130        let member = team
2131            .members
2132            .iter()
2133            .find(|member| member.id == member_id)
2134            .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2135            .clone();
2136        if member.status != TeamMemberStatus::Running {
2137            return Ok(None);
2138        }
2139        let Some(turn_id) = member.current_turn_id.clone() else {
2140            return Ok(None);
2141        };
2142        if self
2143            .active_turn_for_thread(&member.thread_id)
2144            .await
2145            .as_ref()
2146            != Some(&turn_id)
2147        {
2148            return Ok(None);
2149        }
2150        self.interrupt_turn(member.thread_id.clone(), turn_id.clone())
2151            .await?;
2152        // Make queued mailbox work immediately eligible for the replacement turn while the
2153        // lifecycle lock still prevents a follow-up from starting. Delivery acknowledgements
2154        // are turn-owned, so a late acknowledgement from the aborted turn cannot steal a
2155        // reservation acquired by its replacement.
2156        self.teams
2157            .release_mailbox_reservations_for_turn(&turn_id)
2158            .await;
2159        self.teams
2160            .update_member(team_id, member_id, |member| {
2161                member.status = TeamMemberStatus::Interrupted;
2162                member.current_turn_id = None;
2163                member.terminal_error = None;
2164            })
2165            .await?;
2166        self.emit(RoderEvent::TeamMemberCompleted(TeamMemberCompleted {
2167            team_id: team_id.to_string(),
2168            member_id: member_id.to_string(),
2169            member_thread_id: member.thread_id,
2170            turn_id: Some(turn_id.clone()),
2171            status: TeamMemberStatus::Interrupted,
2172            final_message: member.final_message,
2173            error: None,
2174            timestamp: OffsetDateTime::now_utc(),
2175        }))
2176        .await;
2177        Ok(Some(turn_id))
2178    }
2179
2180    pub async fn close_team_member(
2181        &self,
2182        team_id: &str,
2183        member_id: &str,
2184    ) -> anyhow::Result<roder_api::teams::TeamMemberDescriptor> {
2185        let team = self
2186            .read_team(team_id)
2187            .await
2188            .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2189        let member = team
2190            .members
2191            .iter()
2192            .find(|member| member.id == member_id)
2193            .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2194            .clone();
2195        if member.role == roder_api::teams::TeamMemberRole::Lead {
2196            anyhow::bail!("team lead cannot be closed as a subagent");
2197        }
2198        let interrupted_turn_id = if member.status == TeamMemberStatus::Running {
2199            if let Some(turn_id) = member.current_turn_id.clone() {
2200                self.interrupt_turn(member.thread_id.clone(), turn_id.clone())
2201                    .await?;
2202                Some(turn_id)
2203            } else {
2204                None
2205            }
2206        } else {
2207            member.current_turn_id.clone()
2208        };
2209        let updated = self
2210            .teams
2211            .update_member(team_id, member_id, |member| {
2212                member.status = TeamMemberStatus::Closed;
2213                member.current_turn_id = None;
2214            })
2215            .await?;
2216        let closed = updated
2217            .members
2218            .iter()
2219            .find(|member| member.id == member_id)
2220            .cloned()
2221            .ok_or_else(|| anyhow::anyhow!("closed team member disappeared"))?;
2222        self.team_member_turn_contexts
2223            .lock()
2224            .await
2225            .remove(&closed.thread_id);
2226        self.emit(RoderEvent::TeamMemberCompleted(TeamMemberCompleted {
2227            team_id: team_id.to_string(),
2228            member_id: closed.id.clone(),
2229            member_thread_id: closed.thread_id.clone(),
2230            turn_id: interrupted_turn_id,
2231            status: TeamMemberStatus::Closed,
2232            final_message: closed.final_message.clone(),
2233            error: closed.terminal_error.clone(),
2234            timestamp: OffsetDateTime::now_utc(),
2235        }))
2236        .await;
2237        Ok(closed)
2238    }
2239
2240    pub async fn cleanup_team(&self, team_id: &str, force: bool) -> anyhow::Result<bool> {
2241        let Some(team) = self.read_team(team_id).await else {
2242            return Ok(false);
2243        };
2244        if !force
2245            && team
2246                .members
2247                .iter()
2248                .any(|member| member.status == TeamMemberStatus::Running)
2249        {
2250            anyhow::bail!("team {team_id:?} has active teammates; use forced cleanup");
2251        }
2252        if force {
2253            for member in team.members.iter().filter(|member| {
2254                member.role != roder_api::teams::TeamMemberRole::Lead
2255                    && member.status == TeamMemberStatus::Running
2256            }) {
2257                if let Some(turn_id) = member
2258                    .current_turn_id
2259                    .clone()
2260                    .or(self.active_turn_for_thread(&member.thread_id).await)
2261                {
2262                    let _ = self.interrupt_turn(member.thread_id.clone(), turn_id).await;
2263                }
2264            }
2265        }
2266        let removed = self.teams.remove(team_id).await?.is_some();
2267        if removed {
2268            let member_thread_ids = team
2269                .members
2270                .iter()
2271                .map(|member| member.thread_id.clone())
2272                .collect::<std::collections::HashSet<_>>();
2273            self.team_member_turn_contexts
2274                .lock()
2275                .await
2276                .retain(|thread_id, _| !member_thread_ids.contains(thread_id));
2277            self.emit(RoderEvent::TeamCleanupCompleted(TeamCleanupCompleted {
2278                team_id: team_id.to_string(),
2279                forced: force,
2280                timestamp: OffsetDateTime::now_utc(),
2281            }))
2282            .await;
2283        }
2284        Ok(removed)
2285    }
2286
2287    pub async fn effective_policy_mode_for_thread(&self, thread_id: &str) -> PolicyMode {
2288        if let Some(mode) = self.teams.policy_mode_for_thread(thread_id).await {
2289            return mode;
2290        }
2291        self.status().await.policy_mode
2292    }
2293
2294    async fn complete_team_member_turn_with_result(
2295        &self,
2296        thread_id: &ThreadId,
2297        turn_id: &TurnId,
2298        status: TeamMemberStatus,
2299        final_message: Option<String>,
2300        terminal_error: Option<String>,
2301    ) -> anyhow::Result<()> {
2302        let _delivery_guard = self.agent_team_spawn_lock.lock().await;
2303        let Some((team_id, member)) = self
2304            .teams
2305            .complete_member_turn(
2306                thread_id,
2307                turn_id,
2308                status,
2309                final_message.clone(),
2310                terminal_error.clone(),
2311            )
2312            .await?
2313        else {
2314            return Ok(());
2315        };
2316        if let Some(parent_thread_id) = member.parent_thread_id.as_ref()
2317            && let Some(team) = self.read_team(&team_id).await
2318            && let Some(parent) = team
2319                .members
2320                .iter()
2321                .find(|candidate| candidate.thread_id == *parent_thread_id)
2322        {
2323            let identity = member
2324                .agent_path
2325                .as_deref()
2326                .or(member.task_name.as_deref())
2327                .unwrap_or(&member.name);
2328            let mut report = format!("Agent {identity} finished with status {status:?}.");
2329            if let Some(message) = final_message.as_deref()
2330                && !message.trim().is_empty()
2331            {
2332                report.push_str("\n\nFinal result:\n");
2333                report.push_str(message);
2334            }
2335            if let Some(error) = terminal_error.as_deref()
2336                && !error.trim().is_empty()
2337            {
2338                report.push_str("\n\nTerminal error:\n");
2339                report.push_str(error);
2340            }
2341            let mailbox_appended = self
2342                .teams
2343                .append_mailbox_message(
2344                    &team_id,
2345                    Some(member.id.clone()),
2346                    parent.id.clone(),
2347                    TeamMailboxMessageKind::FinalAnswer,
2348                    report,
2349                )
2350                .await
2351                .is_ok();
2352            if mailbox_appended
2353                && let Some(parent_turn_id) = self.active_turn_for_thread(parent_thread_id).await
2354            {
2355                // The parent may finish between the active-turn lookup and enqueue. Its durable
2356                // mailbox entry remains pending for the next turn, while terminal observers must
2357                // still receive TeamMemberCompleted for this child.
2358                let _ = self
2359                    .deliver_pending_team_mailbox(
2360                        &team_id,
2361                        &parent.id,
2362                        parent_thread_id.clone(),
2363                        parent_turn_id,
2364                    )
2365                    .await;
2366            }
2367        }
2368        self.emit(RoderEvent::TeamMemberCompleted(TeamMemberCompleted {
2369            team_id,
2370            member_id: member.id,
2371            member_thread_id: member.thread_id,
2372            turn_id: Some(turn_id.clone()),
2373            status,
2374            final_message,
2375            error: terminal_error,
2376            timestamp: OffsetDateTime::now_utc(),
2377        }))
2378        .await;
2379        Ok(())
2380    }
2381
2382    /// Stops accepting new turns, requests interruption for every active turn,
2383    /// and waits for their runtime tasks to reach terminal cleanup up to
2384    /// `timeout`. Repeated calls are safe and continue draining the same set.
2385    pub async fn drain_active_turns(&self, timeout: std::time::Duration) -> RuntimeDrainOutcome {
2386        let started_at = tokio::time::Instant::now();
2387        let persistence_failures_before =
2388            self.lifecycle_persistence_failures.load(Ordering::Acquire);
2389        let turn_admission = self.turn_admission.lock().await;
2390        self.accepting_turns.store(false, Ordering::Release);
2391        let active = self
2392            .active_turns
2393            .read()
2394            .await
2395            .iter()
2396            .map(|(turn_id, handle)| (turn_id.clone(), handle.thread_id.clone()))
2397            .collect::<Vec<_>>();
2398        let draining = self
2399            .turn_drains
2400            .read()
2401            .await
2402            .iter()
2403            .map(|(turn_id, drain)| (turn_id.clone(), drain.thread_id.clone()))
2404            .collect::<Vec<_>>();
2405        drop(turn_admission);
2406
2407        let mut interrupted_turns = std::collections::BTreeMap::new();
2408        for (turn_id, thread_id) in active.iter().chain(draining.iter()) {
2409            interrupted_turns.insert(turn_id.clone(), thread_id.clone());
2410        }
2411        let interrupted_turn_ids = interrupted_turns.keys().cloned().collect::<Vec<_>>();
2412        for (turn_id, thread_id) in active {
2413            let _ = self
2414                .interrupt_turn_with_reason(thread_id, turn_id, TurnLifecycleReason::Shutdown)
2415                .await;
2416        }
2417
2418        let wait_for_empty = async {
2419            loop {
2420                let notified = self.active_turns_changed.notified();
2421                if self.active_turns.read().await.is_empty()
2422                    && self.turn_drains.read().await.is_empty()
2423                {
2424                    return;
2425                }
2426                notified.await;
2427            }
2428        };
2429        let outcome = if tokio::time::timeout(timeout, wait_for_empty).await.is_ok() {
2430            if self.lifecycle_persistence_failures.load(Ordering::Acquire)
2431                > persistence_failures_before
2432            {
2433                RuntimeDrainOutcome::PersistenceFailed {
2434                    interrupted_turn_ids,
2435                    remaining_turn_ids: Vec::new(),
2436                }
2437            } else {
2438                RuntimeDrainOutcome::Clean {
2439                    interrupted_turn_ids,
2440                }
2441            }
2442        } else {
2443            let active_remaining = self
2444                .active_turns
2445                .read()
2446                .await
2447                .iter()
2448                .map(|(turn_id, handle)| (turn_id.clone(), handle.drain.clone()))
2449                .collect::<Vec<_>>();
2450            let draining_remaining = self
2451                .turn_drains
2452                .read()
2453                .await
2454                .iter()
2455                .map(|(turn_id, drain)| (turn_id.clone(), drain.clone()))
2456                .collect::<Vec<_>>();
2457            let remaining = active_remaining
2458                .into_iter()
2459                .chain(draining_remaining)
2460                .collect::<std::collections::BTreeMap<_, _>>();
2461            let remaining_turn_ids = remaining.keys().cloned().collect::<Vec<_>>();
2462            for turn_id in &remaining_turn_ids {
2463                let handle = remaining
2464                    .get(turn_id)
2465                    .expect("remaining turn ID must have a drain handle");
2466                self.record_turn_lifecycle(
2467                    handle.thread_id.clone(),
2468                    turn_id.clone(),
2469                    TurnLifecycleState::InterruptRequested,
2470                    TurnCleanupState::TimedOut,
2471                    Some(TurnLifecycleReason::Shutdown),
2472                )
2473                .await;
2474            }
2475            if self.lifecycle_persistence_failures.load(Ordering::Acquire)
2476                > persistence_failures_before
2477            {
2478                RuntimeDrainOutcome::PersistenceFailed {
2479                    interrupted_turn_ids,
2480                    remaining_turn_ids,
2481                }
2482            } else {
2483                RuntimeDrainOutcome::DeadlineExceeded {
2484                    interrupted_turn_ids,
2485                    remaining_turn_ids,
2486                }
2487            }
2488        };
2489        self.record_lifecycle_drain_outcome(started_at, &outcome);
2490        outcome
2491    }
2492
2493    /// Enables turn submission after a prior drain attempt. This is intended
2494    /// for embedders that keep a runtime alive after a bounded drain timeout.
2495    pub fn resume_accepting_turns(&self) {
2496        self.accepting_turns.store(true, Ordering::Release);
2497    }
2498
2499    pub async fn turn_lifecycle_snapshot(
2500        &self,
2501        thread_id: &ThreadId,
2502    ) -> anyhow::Result<TurnLifecycleSnapshot> {
2503        let Some(store) = &self.thread_store else {
2504            return Ok(TurnLifecycleSnapshot::default());
2505        };
2506        let extension_states = store.load_extension_states(thread_id).await?;
2507        Ok(turn_lifecycle_snapshot(&extension_states))
2508    }
2509
2510    pub async fn load_thread(
2511        &self,
2512        thread_id: &ThreadId,
2513    ) -> anyhow::Result<Option<ThreadSnapshot>> {
2514        let loaded = if let Some(store) = &self.thread_store {
2515            store.load_thread(thread_id).await?
2516        } else {
2517            None
2518        };
2519        if let Some(snapshot) = loaded.as_ref() {
2520            let live_turn_ids = self
2521                .active_turns
2522                .read()
2523                .await
2524                .keys()
2525                .cloned()
2526                .chain(self.turn_drains.read().await.keys().cloned())
2527                .collect::<std::collections::HashSet<_>>();
2528            let lifecycle = turn_lifecycle_snapshot(&snapshot.extension_states);
2529            for record in lifecycle.records.into_iter().filter(|record| {
2530                record.state.requires_recovery() && !live_turn_ids.contains(&record.turn_id)
2531            }) {
2532                self.lifecycle_restart_reconciliations
2533                    .fetch_add(1, Ordering::AcqRel);
2534                self.record_turn_lifecycle(
2535                    record.thread_id,
2536                    record.turn_id,
2537                    TurnLifecycleState::RecoveryNeeded,
2538                    TurnCleanupState::Unknown,
2539                    Some(TurnLifecycleReason::RuntimeRestart),
2540                )
2541                .await;
2542            }
2543            self.emit(RoderEvent::ThreadLoaded(ThreadLoaded {
2544                thread_id: thread_id.clone(),
2545                timestamp: OffsetDateTime::now_utc(),
2546            }))
2547            .await;
2548        }
2549        Ok(loaded)
2550    }
2551
2552    pub async fn workspace_for_thread(&self, thread_id: &ThreadId) -> anyhow::Result<String> {
2553        if let Some(store) = &self.thread_store {
2554            let snapshot = store
2555                .load_thread(thread_id)
2556                .await?
2557                .ok_or_else(|| anyhow::anyhow!("thread not found: {thread_id}"));
2558            match snapshot {
2559                Ok(snapshot) => {
2560                    if let Some(metadata) = snapshot.metadata {
2561                        // Fork-backed threads fail closed before any write
2562                        // when their workspace was removed out-of-band.
2563                        if let Some(fork) = &metadata.workspace_fork
2564                            && fork.status == roder_api::forks::ForkStatus::Active
2565                            && !std::path::Path::new(&metadata.workspace).is_dir()
2566                        {
2567                            anyhow::bail!(
2568                                "workspace fork {} is missing its workspace at {}; restore it or \
2569                                 remove the fork before running turns in this thread",
2570                                fork.id,
2571                                metadata.workspace
2572                            );
2573                        }
2574                        return Ok(metadata.workspace);
2575                    }
2576                    eprintln!(
2577                        "thread metadata missing while resolving workspace for {thread_id}; falling back to runtime workspace"
2578                    );
2579                }
2580                Err(err) => {
2581                    eprintln!(
2582                        "thread missing while resolving workspace for {thread_id}: {err}; falling back to runtime workspace"
2583                    );
2584                }
2585            }
2586        }
2587        Ok(self.workspace.display().to_string())
2588    }
2589
2590    async fn selection_mode_for_thread(
2591        &self,
2592        thread_id: &ThreadId,
2593    ) -> anyhow::Result<Option<ModelSelectionMode>> {
2594        let Some(store) = &self.thread_store else {
2595            return Ok(None);
2596        };
2597        Ok(store
2598            .load_thread(thread_id)
2599            .await?
2600            .and_then(|snapshot| snapshot.metadata)
2601            .and_then(|metadata| {
2602                metadata
2603                    .selection_mode
2604                    .or_else(|| match (metadata.provider, metadata.model) {
2605                        (Some(provider), Some(model)) => {
2606                            Some(ModelSelectionMode::manual(provider, model, None))
2607                        }
2608                        _ => None,
2609                    })
2610            }))
2611    }
2612
2613    /// Per-thread tool allowlist, developer instructions, and external tools persisted at thread creation.
2614    pub(crate) async fn thread_turn_overrides(
2615        &self,
2616        thread_id: &ThreadId,
2617    ) -> anyhow::Result<ThreadTurnOverrides> {
2618        let Some(store) = &self.thread_store else {
2619            return Ok(ThreadTurnOverrides::default());
2620        };
2621        Ok(store
2622            .load_thread_metadata(thread_id)
2623            .await?
2624            .map(|metadata| ThreadTurnOverrides {
2625                tool_allowlist: metadata.tool_allowlist,
2626                developer_instructions: metadata.developer_instructions,
2627                external_tools: metadata.external_tools,
2628            })
2629            .unwrap_or_default())
2630    }
2631
2632    pub async fn set_thread_selection_mode(
2633        &self,
2634        thread_id: &ThreadId,
2635        selection_mode: ModelSelectionMode,
2636    ) -> anyhow::Result<()> {
2637        let Some(store) = &self.thread_store else {
2638            return Ok(());
2639        };
2640        let Some(snapshot) = store.load_thread(thread_id).await? else {
2641            anyhow::bail!("thread not found: {thread_id}");
2642        };
2643        let Some(mut metadata) = snapshot.metadata else {
2644            return Ok(());
2645        };
2646        let concrete = selection_mode.concrete_selection();
2647        metadata.provider = Some(concrete.provider);
2648        metadata.model = Some(concrete.model);
2649        metadata.selection_mode = Some(selection_mode);
2650        metadata.updated_at = OffsetDateTime::now_utc();
2651        store.update_thread_metadata(metadata).await?;
2652        Ok(())
2653    }
2654
2655    async fn runner_session_for_thread(
2656        &self,
2657        thread_id: &ThreadId,
2658    ) -> anyhow::Result<Option<(RunnerDestination, Arc<dyn RemoteRunnerSession>)>> {
2659        let metadata = if let Some(store) = &self.thread_store {
2660            store.load_thread_metadata(thread_id).await?
2661        } else {
2662            None
2663        };
2664        // An explicit per-thread binding wins over the runtime-level destination.
2665        let destination = metadata
2666            .as_ref()
2667            .and_then(|metadata| metadata.runner_binding.as_ref())
2668            .map(|binding| binding.destination.clone())
2669            .or(self.config.read().await.remote_runner_destination.clone());
2670        let Some(destination) = destination else {
2671            self.evict_runner_session(thread_id).await;
2672            return Ok(None);
2673        };
2674        let provider = self
2675            .registry
2676            .remote_runner_providers
2677            .iter()
2678            .find(|provider| provider.id() == destination.provider_id)
2679            .cloned()
2680            .ok_or_else(|| {
2681                anyhow::anyhow!(
2682                    "remote runner provider {:?} is not installed",
2683                    destination.provider_id
2684                )
2685            })?;
2686        let persisted_state = metadata.and_then(|metadata| metadata.runner_state);
2687        let slot = {
2688            let mut sessions = self.runner_sessions.lock().await;
2689            match sessions.get(thread_id) {
2690                Some(slot) if slot.matches(&destination) => slot.clone(),
2691                _ => {
2692                    let slot = Arc::new(RunnerSessionSlot::empty(&destination));
2693                    sessions.insert(thread_id.clone(), slot.clone());
2694                    slot
2695                }
2696            }
2697        };
2698
2699        loop {
2700            let initialized = slot.initialized.notified();
2701            let mut state = slot.state.lock().await;
2702            match &*state {
2703                RunnerSessionSlotState::Ready(cached) => {
2704                    return Ok(Some((cached.destination.clone(), cached.session.clone())));
2705                }
2706                RunnerSessionSlotState::Failed(error) => {
2707                    return Err(anyhow::anyhow!(error.to_string()));
2708                }
2709                RunnerSessionSlotState::Initializing => {
2710                    drop(state);
2711                    initialized.await;
2712                }
2713                RunnerSessionSlotState::Empty => {
2714                    *state = RunnerSessionSlotState::Initializing;
2715                    drop(state);
2716                    self.spawn_runner_session_initialization(
2717                        thread_id.clone(),
2718                        destination.clone(),
2719                        provider.clone(),
2720                        persisted_state.clone(),
2721                        slot.clone(),
2722                    );
2723                }
2724            }
2725        }
2726    }
2727
2728    fn spawn_runner_session_initialization(
2729        &self,
2730        thread_id: ThreadId,
2731        destination: RunnerDestination,
2732        provider: Arc<dyn RemoteRunnerProvider>,
2733        persisted_state: Option<RunnerSessionState>,
2734        slot: Arc<RunnerSessionSlot>,
2735    ) {
2736        let thread_store = self.thread_store.clone();
2737        let runner_sessions = self.runner_sessions.clone();
2738        // The task intentionally outlives the turn that first requested the
2739        // workspace. Interrupting that turn must not strand the slot in
2740        // `Initializing` and deadlock every later tool or lifecycle call.
2741        drop(tokio::spawn(async move {
2742            let initialized = async {
2743                let session = if let Some(state) = persisted_state
2744                    && state.provider_id == destination.provider_id
2745                    && state.destination_id == destination.id
2746                {
2747                    match provider.resume_session(state).await {
2748                        Ok(session) => session,
2749                        Err(_) => provider.create_session(destination.clone()).await?,
2750                    }
2751                } else {
2752                    provider.create_session(destination.clone()).await?
2753                };
2754                let resolved = (destination.clone(), session.clone());
2755                // Persist before releasing the singleflight or dispatching
2756                // the first tool. A later process can rejoin even if the turn
2757                // that requested initialization has already been interrupted.
2758                Self::persist_runner_state_in_store(
2759                    thread_store.as_ref(),
2760                    &thread_id,
2761                    Some(&resolved),
2762                )
2763                .await?;
2764                Ok::<_, anyhow::Error>(CachedRunnerSession {
2765                    destination,
2766                    session,
2767                })
2768            }
2769            .await;
2770
2771            match initialized {
2772                Ok(cached) => {
2773                    *slot.state.lock().await = RunnerSessionSlotState::Ready(cached);
2774                    slot.initialized.notify_waiters();
2775                }
2776                Err(error) => {
2777                    let message: Arc<str> = Arc::from(format!("{error:#}"));
2778                    *slot.state.lock().await = RunnerSessionSlotState::Failed(message);
2779                    slot.initialized.notify_waiters();
2780                    let mut sessions = runner_sessions.lock().await;
2781                    let is_current = sessions
2782                        .get(&thread_id)
2783                        .is_some_and(|current| Arc::ptr_eq(current, &slot));
2784                    if is_current {
2785                        sessions.remove(&thread_id);
2786                    }
2787                }
2788            }
2789        }));
2790    }
2791
2792    async fn evict_runner_session(&self, thread_id: &ThreadId) {
2793        self.runner_sessions.lock().await.remove(thread_id);
2794    }
2795
2796    async fn cache_runner_session(
2797        &self,
2798        thread_id: &ThreadId,
2799        destination: RunnerDestination,
2800        session: Arc<dyn RemoteRunnerSession>,
2801    ) {
2802        let cached = CachedRunnerSession {
2803            destination,
2804            session,
2805        };
2806        self.runner_sessions.lock().await.insert(
2807            thread_id.clone(),
2808            Arc::new(RunnerSessionSlot::ready(cached)),
2809        );
2810    }
2811
2812    /// Read a thread's explicit runner binding without creating or resuming a
2813    /// session. Tool routing uses this before local previews so runner-backed
2814    /// and fail-closed hosted calls never inspect the host workspace.
2815    pub(crate) async fn runner_binding_for_thread(
2816        &self,
2817        thread_id: &ThreadId,
2818    ) -> anyhow::Result<Option<ThreadRunnerBinding>> {
2819        let Some(store) = &self.thread_store else {
2820            return Ok(None);
2821        };
2822        Ok(store
2823            .load_thread_metadata(thread_id)
2824            .await?
2825            .and_then(|metadata| metadata.runner_binding))
2826    }
2827
2828    /// Resolve a previously-read binding into a live remote workspace. The
2829    /// session remains lazy: callers invoke this only after policy approval.
2830    pub(crate) async fn remote_workspace_for_binding(
2831        &self,
2832        thread_id: &ThreadId,
2833        binding: ThreadRunnerBinding,
2834    ) -> anyhow::Result<Arc<RemoteWorkspace>> {
2835        let session = self
2836            .runner_session_for_thread(thread_id)
2837            .await?
2838            .map(|(_, session)| session)
2839            .ok_or_else(|| {
2840                anyhow::anyhow!("runner-bound thread {thread_id} has no runner session")
2841            })?;
2842        Ok(Arc::new(RemoteWorkspace {
2843            session,
2844            root: binding.workspace,
2845            read_roots: binding.read_roots,
2846        }))
2847    }
2848
2849    pub(crate) async fn runner_tool_execution_lock(
2850        &self,
2851        session: &dyn RemoteRunnerSession,
2852    ) -> Arc<RunnerToolExecutionMutex> {
2853        let state = session.state();
2854        let key = (state.provider_id, state.session_id);
2855        let mut locks = self.runner_tool_execution_locks.lock().await;
2856        if let Some(lock) = locks.get(&key).and_then(Weak::upgrade) {
2857            return lock;
2858        }
2859
2860        locks.retain(|_, lock| lock.strong_count() > 0);
2861        let lock = Arc::new(Mutex::new(()));
2862        locks.insert(key, Arc::downgrade(&lock));
2863        lock
2864    }
2865
2866    async fn persist_runner_state(
2867        &self,
2868        thread_id: &ThreadId,
2869        runner: Option<&(RunnerDestination, Arc<dyn RemoteRunnerSession>)>,
2870    ) -> anyhow::Result<()> {
2871        Self::persist_runner_state_in_store(self.thread_store.as_ref(), thread_id, runner).await
2872    }
2873
2874    async fn persist_runner_state_in_store(
2875        store: Option<&Arc<dyn ThreadStore>>,
2876        thread_id: &ThreadId,
2877        runner: Option<&(RunnerDestination, Arc<dyn RemoteRunnerSession>)>,
2878    ) -> anyhow::Result<()> {
2879        let Some((destination, session)) = runner else {
2880            return Ok(());
2881        };
2882        let Some(store) = store else {
2883            return Ok(());
2884        };
2885        let Some(snapshot) = store.load_thread(thread_id).await? else {
2886            return Ok(());
2887        };
2888        let Some(mut metadata) = snapshot.metadata else {
2889            return Ok(());
2890        };
2891        metadata.runner_destination = Some(destination.clone());
2892        metadata.runner_state = Some(session.state());
2893        metadata.updated_at = OffsetDateTime::now_utc();
2894        store.update_thread_metadata(metadata).await?;
2895        Ok(())
2896    }
2897
2898    fn remote_runner_provider_by_id(
2899        &self,
2900        provider_id: &str,
2901    ) -> Option<Arc<dyn RemoteRunnerProvider>> {
2902        self.registry
2903            .remote_runner_providers
2904            .iter()
2905            .find(|provider| provider.id() == provider_id)
2906            .cloned()
2907    }
2908
2909    /// Resolve the live runner session for a thread along with its provider,
2910    /// failing clearly when the thread is not runner-bound.
2911    async fn thread_runner_session(
2912        &self,
2913        thread_id: &ThreadId,
2914    ) -> anyhow::Result<(
2915        RunnerDestination,
2916        Arc<dyn RemoteRunnerProvider>,
2917        Arc<dyn RemoteRunnerSession>,
2918    )> {
2919        let Some((destination, session)) = self.runner_session_for_thread(thread_id).await? else {
2920            anyhow::bail!("thread {thread_id} is not bound to a remote runner");
2921        };
2922        let provider = self
2923            .remote_runner_provider_by_id(&destination.provider_id)
2924            .ok_or_else(|| {
2925                anyhow::anyhow!(
2926                    "remote runner provider {:?} is not installed",
2927                    destination.provider_id
2928                )
2929            })?;
2930        Ok((destination, provider, session))
2931    }
2932
2933    /// Pause a runner-bound thread's session toward standby and persist the
2934    /// post-pause state. Errors if the provider is not pausable.
2935    pub async fn pause_thread_runner(
2936        &self,
2937        thread_id: &ThreadId,
2938    ) -> anyhow::Result<RunnerSessionState> {
2939        let (destination, provider, session) = self.thread_runner_session(thread_id).await?;
2940        anyhow::ensure!(
2941            provider.capabilities().pausable,
2942            "remote runner provider {:?} does not support pausing",
2943            destination.provider_id
2944        );
2945        let state = session.pause().await?;
2946        self.persist_runner_state(thread_id, Some(&(destination, session)))
2947            .await?;
2948        Ok(state)
2949    }
2950
2951    /// Resume (wake) a runner-bound thread's paused session and persist state.
2952    pub async fn resume_thread_runner(
2953        &self,
2954        thread_id: &ThreadId,
2955    ) -> anyhow::Result<RunnerSessionState> {
2956        let (destination, _provider, session) = self.thread_runner_session(thread_id).await?;
2957        let state = session.resume().await?;
2958        self.persist_runner_state(thread_id, Some(&(destination, session)))
2959            .await?;
2960        Ok(state)
2961    }
2962
2963    /// Detach a runner-bound thread's session: persist the durable, rejoinable
2964    /// state and leave the remote sandbox alive. Errors if not detachable.
2965    pub async fn detach_thread_runner(
2966        &self,
2967        thread_id: &ThreadId,
2968    ) -> anyhow::Result<RunnerSessionState> {
2969        let (destination, provider, session) = self.thread_runner_session(thread_id).await?;
2970        anyhow::ensure!(
2971            provider.capabilities().detachable,
2972            "remote runner provider {:?} does not support detaching",
2973            destination.provider_id
2974        );
2975        let state = session.detach().await?;
2976        // Persist the detached state explicitly so a later turn or process
2977        // rejoins the same sandbox instead of provisioning a new one.
2978        if let Some(store) = &self.thread_store
2979            && let Some(snapshot) = store.load_thread(thread_id).await?
2980            && let Some(mut metadata) = snapshot.metadata
2981        {
2982            metadata.runner_destination = Some(destination);
2983            metadata.runner_state = Some(state.clone());
2984            metadata.updated_at = OffsetDateTime::now_utc();
2985            store.update_thread_metadata(metadata).await?;
2986        }
2987        self.evict_runner_session(thread_id).await;
2988        Ok(state)
2989    }
2990
2991    /// Rejoin a previously created sandbox from a thread's persisted runner
2992    /// state without provisioning a new one. An optional `sandbox` overrides the
2993    /// persisted sandbox name (recovery by name). Persists refreshed state.
2994    pub async fn rejoin_thread_runner(
2995        &self,
2996        thread_id: &ThreadId,
2997        sandbox: Option<String>,
2998    ) -> anyhow::Result<RunnerSessionState> {
2999        let store = self
3000            .thread_store
3001            .as_ref()
3002            .ok_or_else(|| anyhow::anyhow!("thread store is required to rejoin a runner"))?;
3003        let metadata = store
3004            .load_thread_metadata(thread_id)
3005            .await?
3006            .ok_or_else(|| anyhow::anyhow!("thread {thread_id} has no metadata"))?;
3007        let destination = metadata
3008            .runner_binding
3009            .as_ref()
3010            .map(|binding| binding.destination.clone())
3011            .or_else(|| metadata.runner_destination.clone())
3012            .ok_or_else(|| anyhow::anyhow!("thread {thread_id} is not bound to a remote runner"))?;
3013        let mut state = metadata
3014            .runner_state
3015            .clone()
3016            .ok_or_else(|| anyhow::anyhow!("thread {thread_id} has no persisted runner state"))?;
3017        if let Some(sandbox) = sandbox
3018            && let Some(object) = state.metadata.as_object_mut()
3019        {
3020            object.insert("sandbox_name".to_string(), serde_json::Value::from(sandbox));
3021        }
3022        let provider = self
3023            .remote_runner_provider_by_id(&destination.provider_id)
3024            .ok_or_else(|| {
3025                anyhow::anyhow!(
3026                    "remote runner provider {:?} is not installed",
3027                    destination.provider_id
3028                )
3029            })?;
3030        let session = provider.rejoin_session(state).await?;
3031        self.persist_runner_state(thread_id, Some(&(destination.clone(), session.clone())))
3032            .await?;
3033        self.cache_runner_session(thread_id, destination, session.clone())
3034            .await;
3035        Ok(session.state())
3036    }
3037
3038    async fn record_thread_usage_metadata(
3039        &self,
3040        thread_id: &ThreadId,
3041        usage: &TokenUsage,
3042    ) -> anyhow::Result<()> {
3043        if usage.is_empty() {
3044            return Ok(());
3045        }
3046        let Some(store) = &self.thread_store else {
3047            return Ok(());
3048        };
3049        let Some(snapshot) = store.load_thread(thread_id).await? else {
3050            return Ok(());
3051        };
3052        let Some(mut metadata) = snapshot.metadata else {
3053            return Ok(());
3054        };
3055        metadata
3056            .usage
3057            .get_or_insert_with(ThreadUsageMetadata::default)
3058            .add_token_usage(usage);
3059        metadata.updated_at = OffsetDateTime::now_utc();
3060        store.update_thread_metadata(metadata).await?;
3061        Ok(())
3062    }
3063
3064    pub fn start_turn(
3065        self: &Arc<Self>,
3066        mut req: StartTurnRequest,
3067    ) -> BoxFuture<'_, anyhow::Result<TurnId>> {
3068        Box::pin(async move {
3069            let turn_admission = self.turn_admission.lock().await;
3070            anyhow::ensure!(
3071                self.accepting_turns.load(Ordering::Acquire),
3072                "runtime is quiescing and cannot accept new turns"
3073            );
3074            req.workspace = validate_thread_workspace(&req.workspace)?;
3075            let team_member = self.teams.member_for_thread(&req.thread_id).await;
3076            let cfg = self.config.read().await.clone();
3077            let provider = req
3078                .provider_override
3079                .clone()
3080                .unwrap_or_else(|| cfg.default_provider.clone());
3081            self.engine_for(&provider)?;
3082            let turn_id = uuid::Uuid::new_v4().to_string();
3083            let mut initial_mailbox_ack = None;
3084            if let Some((team_id, member)) = &team_member {
3085                let pending = self
3086                    .teams
3087                    .reserve_pending_mailbox_messages(team_id, &member.id, &turn_id)
3088                    .await?;
3089                if !pending.is_empty()
3090                    && let Some(team) = self.read_team(team_id).await
3091                {
3092                    let mailbox = format_mailbox_messages(&team, &pending);
3093                    req.message = if req.message.trim().is_empty() {
3094                        mailbox
3095                    } else {
3096                        format!("{mailbox}\n\n[Direct task input]\n{}", req.message)
3097                    };
3098                    initial_mailbox_ack = Some(MailboxDeliveryAck {
3099                        team_id: team_id.clone(),
3100                        message_ids: pending.iter().map(|message| message.id.clone()).collect(),
3101                    });
3102                }
3103            }
3104            let (abort_handle, abort_registration) = AbortHandle::new_pair();
3105            let drain = Arc::new(TurnDrainHandle {
3106                thread_id: req.thread_id.clone(),
3107                interrupt_requested: AtomicBool::new(false),
3108                interrupt_reason: Mutex::new(None),
3109                completed: AtomicBool::new(false),
3110                completed_notify: Notify::new(),
3111            });
3112            let active = ActiveTurnHandle {
3113                thread_id: req.thread_id.clone(),
3114                abort: abort_handle,
3115                steers: Arc::new(Mutex::new(Vec::new())),
3116                drain,
3117            };
3118            self.active_turns
3119                .write()
3120                .await
3121                .insert(turn_id.clone(), active);
3122            self.record_turn_lifecycle(
3123                req.thread_id.clone(),
3124                turn_id.clone(),
3125                TurnLifecycleState::Running,
3126                TurnCleanupState::NotRequested,
3127                None,
3128            )
3129            .await;
3130            self.active_turn_contexts.write().await.insert(
3131                turn_id.clone(),
3132                InheritedTurnContext {
3133                    workspace: req.workspace.clone(),
3134                    instructions: req.instructions.clone(),
3135                    developer_context: req.developer_context.clone(),
3136                },
3137            );
3138            if let Some((team_id, member)) = team_member {
3139                let updated = match self
3140                    .teams
3141                    .update_member(&team_id, &member.id, |member| {
3142                        member.current_turn_id = Some(turn_id.clone());
3143                        member.status = TeamMemberStatus::Running;
3144                        member.final_message = None;
3145                        member.terminal_error = None;
3146                    })
3147                    .await
3148                {
3149                    Ok(updated) => updated,
3150                    Err(error) => {
3151                        self.active_turns.write().await.remove(&turn_id);
3152                        self.active_turn_contexts.write().await.remove(&turn_id);
3153                        self.teams
3154                            .release_mailbox_reservations_for_turn(&turn_id)
3155                            .await;
3156                        return Err(error);
3157                    }
3158                };
3159                if let Some(member) = updated
3160                    .members
3161                    .into_iter()
3162                    .find(|candidate| candidate.id == member.id)
3163                {
3164                    self.emit(RoderEvent::TeamMemberStatusChanged(
3165                        TeamMemberStatusChanged {
3166                            team_id,
3167                            member_id: member.id,
3168                            member_thread_id: member.thread_id,
3169                            status: TeamMemberStatus::Running,
3170                            timestamp: OffsetDateTime::now_utc(),
3171                        },
3172                    ))
3173                    .await;
3174                }
3175            }
3176            let runtime = Arc::clone(self);
3177            let turn_req = req;
3178            let thread_id_for_task = turn_req.thread_id.clone();
3179            let turn_id_for_task = turn_id.clone();
3180            tokio::spawn(async move {
3181                let result = Abortable::new(
3182                    runtime.run_turn(turn_req, turn_id_for_task.clone(), initial_mailbox_ack),
3183                    abort_registration,
3184                )
3185                .await;
3186                /*
3187                 * A failed sibling in a parallel tool batch drops in-flight external tool
3188                 * futures (`try_join_all` in `route_tool_calls`), stranding their
3189                 * `pending_external_tool_calls` entries. Sweep before reporting the turn
3190                 * outcome so every `thread/toolExecutionRequested` gets a terminal
3191                 * resolution; on clean completion the map holds nothing for this turn.
3192                 */
3193                runtime
3194                    .cancel_pending_external_tool_calls_for_turn(&turn_id_for_task)
3195                    .await;
3196                let completed = matches!(&result, Ok(Ok(TurnRunOutcome::Completed)));
3197                match &result {
3198                    Ok(Err(err)) => {
3199                        let (cleanup, ownership) =
3200                            runtime.await_provider_turn_cleanup(&turn_id_for_task).await;
3201                        // run_turn emits failures after the stream starts; this covers setup/startup errors.
3202                        runtime
3203                            .emit(RoderEvent::TurnFailed(TurnFailed {
3204                                thread_id: thread_id_for_task.clone(),
3205                                turn_id: turn_id_for_task.clone(),
3206                                error: err.to_string(),
3207                                error_kind: None,
3208                                usage: None,
3209                                timestamp: OffsetDateTime::now_utc(),
3210                            }))
3211                            .await;
3212                        runtime
3213                            .record_turn_lifecycle_with_ownership(
3214                                thread_id_for_task.clone(),
3215                                turn_id_for_task.clone(),
3216                                TurnLifecycleState::Failed,
3217                                cleanup,
3218                                Some(TurnLifecycleReason::ProviderFailure),
3219                                ownership,
3220                            )
3221                            .await;
3222                        let _ = runtime
3223                            .complete_team_member_turn_with_result(
3224                                &thread_id_for_task,
3225                                &turn_id_for_task,
3226                                TeamMemberStatus::Failed,
3227                                None,
3228                                Some(err.to_string()),
3229                            )
3230                            .await;
3231                    }
3232                    Ok(Ok(TurnRunOutcome::Stopped)) => {
3233                        let (cleanup, ownership) =
3234                            runtime.await_provider_turn_cleanup(&turn_id_for_task).await;
3235                        runtime
3236                            .record_turn_lifecycle_with_ownership(
3237                                thread_id_for_task.clone(),
3238                                turn_id_for_task.clone(),
3239                                TurnLifecycleState::Failed,
3240                                cleanup,
3241                                Some(TurnLifecycleReason::ProviderFailure),
3242                                ownership,
3243                            )
3244                            .await;
3245                        let _ = runtime
3246                            .complete_team_member_turn_with_result(
3247                                &thread_id_for_task,
3248                                &turn_id_for_task,
3249                                TeamMemberStatus::Failed,
3250                                None,
3251                                Some("turn stopped before completion".to_string()),
3252                            )
3253                            .await;
3254                    }
3255                    Err(_) => {
3256                        let reason = if let Some(handle) = runtime
3257                            .turn_drains
3258                            .read()
3259                            .await
3260                            .get(&turn_id_for_task)
3261                            .cloned()
3262                        {
3263                            handle
3264                                .interrupt_reason
3265                                .lock()
3266                                .await
3267                                .unwrap_or(TurnLifecycleReason::RuntimeFailure)
3268                        } else {
3269                            TurnLifecycleReason::RuntimeFailure
3270                        };
3271                        let (cleanup, ownership) =
3272                            runtime.await_provider_turn_cleanup(&turn_id_for_task).await;
3273                        runtime
3274                            .record_turn_lifecycle_with_ownership(
3275                                thread_id_for_task.clone(),
3276                                turn_id_for_task.clone(),
3277                                TurnLifecycleState::Interrupted,
3278                                cleanup,
3279                                Some(reason),
3280                                ownership,
3281                            )
3282                            .await;
3283                        runtime
3284                            .emit(RoderEvent::TurnInterrupted(TurnInterrupted {
3285                                thread_id: thread_id_for_task.clone(),
3286                                turn_id: turn_id_for_task.clone(),
3287                                timestamp: OffsetDateTime::now_utc(),
3288                            }))
3289                            .await;
3290                        let _ = runtime
3291                            .complete_team_member_turn_with_result(
3292                                &thread_id_for_task,
3293                                &turn_id_for_task,
3294                                TeamMemberStatus::Interrupted,
3295                                None,
3296                                None,
3297                            )
3298                            .await;
3299                    }
3300                    Ok(Ok(TurnRunOutcome::Completed)) => {}
3301                }
3302                runtime
3303                    .teams
3304                    .release_mailbox_reservations_for_turn(&turn_id_for_task)
3305                    .await;
3306                runtime.active_turns.write().await.remove(&turn_id_for_task);
3307                if let Some(drain) = runtime.turn_drains.write().await.remove(&turn_id_for_task) {
3308                    drain.completed.store(true, Ordering::Release);
3309                    drain.completed_notify.notify_waiters();
3310                }
3311                runtime
3312                    .active_turn_selections
3313                    .write()
3314                    .await
3315                    .remove(&turn_id_for_task);
3316                runtime
3317                    .active_turn_contexts
3318                    .write()
3319                    .await
3320                    .remove(&turn_id_for_task);
3321                if !completed {
3322                    // Completion paths above consume registered provider cleanup
3323                    // handles. A setup failure before an engine can stream has no
3324                    // such handle; this is a harmless final defensive sweep.
3325                    let _ = runtime.provider_turn_cleanups.lock().map(|mut cleanups| {
3326                        cleanups.remove(&turn_id_for_task);
3327                    });
3328                }
3329                runtime.active_turns_changed.notify_waiters();
3330                if completed {
3331                    let _ = runtime
3332                        .continue_active_goal_after_turn(thread_id_for_task)
3333                        .await;
3334                }
3335            });
3336            drop(turn_admission);
3337            Ok(turn_id)
3338        })
3339    }
3340
3341    pub(crate) async fn has_active_turn_for_thread(&self, thread_id: &ThreadId) -> bool {
3342        self.active_turns
3343            .read()
3344            .await
3345            .values()
3346            .any(|handle| &handle.thread_id == thread_id)
3347    }
3348
3349    async fn ensure_codex_v2_team_capacity(
3350        &self,
3351        team: &TeamState,
3352        resuming_member_id: &str,
3353        candidate_model: Option<&str>,
3354    ) -> anyhow::Result<()> {
3355        if !is_codex_v2_team(team)
3356            && !candidate_model
3357                .is_some_and(|model| model_supports_reasoning_effort(model, REASONING_ULTRA))
3358        {
3359            return Ok(());
3360        }
3361        let active_thread_ids = self
3362            .active_turns
3363            .read()
3364            .await
3365            .values()
3366            .map(|handle| handle.thread_id.clone())
3367            .collect::<std::collections::HashSet<_>>();
3368        let resident_threads = 1 + team
3369            .members
3370            .iter()
3371            .filter(|member| {
3372                member.role != roder_api::teams::TeamMemberRole::Lead
3373                    && member.id != resuming_member_id
3374                    && active_thread_ids.contains(&member.thread_id)
3375            })
3376            .count();
3377        anyhow::ensure!(
3378            resident_threads < codex_v2::CODEX_V2_MAX_RESIDENT_TEAM_THREADS,
3379            "agent thread limit reached for this Codex V2 team: maximum {} resident threads (the lead plus 3 running subagents)",
3380            codex_v2::CODEX_V2_MAX_RESIDENT_TEAM_THREADS
3381        );
3382        Ok(())
3383    }
3384
3385    /// Number of currently running turns across all threads. Hosted runtime
3386    /// pools use this to avoid evicting tenants with active work.
3387    pub async fn active_turn_count(&self) -> usize {
3388        self.active_turns.read().await.len()
3389    }
3390
3391    pub async fn active_turn_for_thread(&self, thread_id: &ThreadId) -> Option<TurnId> {
3392        self.active_turns
3393            .read()
3394            .await
3395            .iter()
3396            .find_map(|(turn_id, handle)| (&handle.thread_id == thread_id).then(|| turn_id.clone()))
3397    }
3398
3399    pub async fn thread_activity(&self, thread_id: &ThreadId) -> ThreadActivity {
3400        let active_turn_id = self.active_turn_for_thread(thread_id).await;
3401        let draining_turn_id = if active_turn_id.is_none() {
3402            self.turn_drains
3403                .read()
3404                .await
3405                .iter()
3406                .find_map(|(turn_id, drain)| {
3407                    (&drain.thread_id == thread_id).then(|| turn_id.clone())
3408                })
3409        } else {
3410            None
3411        };
3412        let Some(active_turn_id) = active_turn_id.or(draining_turn_id) else {
3413            return ThreadActivity::default();
3414        };
3415
3416        let mut active_flags = Vec::new();
3417        {
3418            let pending_approvals = self.pending_tool_approvals.lock().await;
3419            if pending_approvals
3420                .values()
3421                .any(|pending| &pending.thread_id == thread_id && pending.turn_id == active_turn_id)
3422            {
3423                active_flags.push("approvalRequired".to_string());
3424            }
3425        }
3426        {
3427            let pending_inputs = self.pending_user_inputs.lock().await;
3428            if pending_inputs
3429                .values()
3430                .any(|pending| &pending.thread_id == thread_id && pending.turn_id == active_turn_id)
3431            {
3432                active_flags.push("userInputRequired".to_string());
3433            }
3434        }
3435        {
3436            let pending_external = self.pending_external_tool_calls.lock().await;
3437            if pending_external
3438                .values()
3439                .any(|pending| &pending.thread_id == thread_id && pending.turn_id == active_turn_id)
3440            {
3441                active_flags.push("externalToolPending".to_string());
3442            }
3443        }
3444        let interrupting = self
3445            .active_turns
3446            .read()
3447            .await
3448            .get(&active_turn_id)
3449            .is_some_and(|handle| handle.drain.interrupt_requested.load(Ordering::Acquire))
3450            || self
3451                .turn_drains
3452                .read()
3453                .await
3454                .get(&active_turn_id)
3455                .is_some_and(|drain| drain.interrupt_requested.load(Ordering::Acquire));
3456        if interrupting {
3457            active_flags.push("interrupting".to_string());
3458        }
3459        if self.pending_plan_exit().await.is_some_and(|pending| {
3460            &pending.thread_id == thread_id && pending.turn_id == active_turn_id
3461        }) {
3462            active_flags.push("planExitRequired".to_string());
3463        }
3464
3465        ThreadActivity {
3466            active_turn_id: Some(active_turn_id),
3467            active_flags,
3468        }
3469    }
3470
3471    pub async fn interrupt_turn(&self, thread_id: ThreadId, turn_id: TurnId) -> anyhow::Result<()> {
3472        self.interrupt_turn_with_reason(thread_id, turn_id, TurnLifecycleReason::UserInterrupt)
3473            .await
3474    }
3475
3476    async fn interrupt_turn_with_reason(
3477        &self,
3478        thread_id: ThreadId,
3479        turn_id: TurnId,
3480        reason: TurnLifecycleReason,
3481    ) -> anyhow::Result<()> {
3482        let handle = self.active_turns.write().await.remove(&turn_id);
3483        if let Some(handle) = handle {
3484            if handle
3485                .drain
3486                .interrupt_requested
3487                .swap(true, Ordering::AcqRel)
3488            {
3489                return Ok(());
3490            }
3491            *handle.drain.interrupt_reason.lock().await = Some(reason);
3492            self.turn_drains
3493                .write()
3494                .await
3495                .insert(turn_id.clone(), handle.drain.clone());
3496            let ownership = self.provider_cleanup_ownership(&turn_id);
3497            self.record_turn_lifecycle_with_ownership(
3498                thread_id.clone(),
3499                turn_id.clone(),
3500                TurnLifecycleState::InterruptRequested,
3501                TurnCleanupState::Requested,
3502                Some(reason),
3503                ownership,
3504            )
3505            .await;
3506            handle.abort.abort();
3507            self.active_turns_changed.notify_waiters();
3508        }
3509        self.active_turn_selections.write().await.remove(&turn_id);
3510        self.cancel_pending_external_tool_calls_for_turn(&turn_id)
3511            .await;
3512        Ok(())
3513    }
3514
3515    pub async fn steer_turn(
3516        &self,
3517        thread_id: ThreadId,
3518        turn_id: TurnId,
3519        message: String,
3520        images: Vec<InputImage>,
3521    ) -> anyhow::Result<()> {
3522        self.enqueue_turn_steer(thread_id, turn_id, message, images, None)
3523            .await
3524    }
3525
3526    pub(crate) async fn has_pending_turn_steers(&self, turn_id: &TurnId) -> bool {
3527        let active = self.active_turns.read().await.get(turn_id).cloned();
3528        let Some(active) = active else {
3529            return false;
3530        };
3531        let has_pending = !active.steers.lock().await.is_empty();
3532        has_pending
3533    }
3534
3535    async fn steer_turn_with_mailbox_ack(
3536        &self,
3537        thread_id: ThreadId,
3538        turn_id: TurnId,
3539        message: String,
3540        message_ids: Vec<String>,
3541        team_id: TeamId,
3542    ) -> anyhow::Result<()> {
3543        self.enqueue_turn_steer(
3544            thread_id,
3545            turn_id,
3546            message,
3547            Vec::new(),
3548            Some(MailboxDeliveryAck {
3549                team_id,
3550                message_ids,
3551            }),
3552        )
3553        .await
3554    }
3555
3556    async fn deliver_pending_team_mailbox(
3557        &self,
3558        team_id: &str,
3559        member_id: &str,
3560        member_thread_id: ThreadId,
3561        turn_id: TurnId,
3562    ) -> anyhow::Result<()> {
3563        let pending = self
3564            .teams
3565            .reserve_pending_mailbox_messages(team_id, member_id, &turn_id)
3566            .await?;
3567        if pending.is_empty() {
3568            return Ok(());
3569        }
3570        let message_ids = pending
3571            .iter()
3572            .map(|message| message.id.clone())
3573            .collect::<Vec<_>>();
3574        let team = self
3575            .read_team(team_id)
3576            .await
3577            .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
3578        if let Err(error) = self
3579            .steer_turn_with_mailbox_ack(
3580                member_thread_id,
3581                turn_id.clone(),
3582                format_mailbox_messages(&team, &pending),
3583                message_ids.clone(),
3584                team_id.to_string(),
3585            )
3586            .await
3587        {
3588            self.teams
3589                .release_mailbox_reservations(&turn_id, &message_ids)
3590                .await;
3591            return Err(error);
3592        }
3593        Ok(())
3594    }
3595
3596    async fn enqueue_turn_steer(
3597        &self,
3598        thread_id: ThreadId,
3599        turn_id: TurnId,
3600        message: String,
3601        images: Vec<InputImage>,
3602        mailbox_ack: Option<MailboxDeliveryAck>,
3603    ) -> anyhow::Result<()> {
3604        let message = message.trim().to_string();
3605        if message.is_empty() && images.is_empty() {
3606            return Ok(());
3607        }
3608
3609        let Some(active) = self.active_turns.read().await.get(&turn_id).cloned() else {
3610            anyhow::bail!("no active turn to steer");
3611        };
3612        active.steers.lock().await.push(QueuedTurnSteer {
3613            message: UserMessage::with_images(message.clone(), images),
3614            mailbox_ack,
3615        });
3616        self.emit(RoderEvent::TurnSteered(TurnSteered {
3617            thread_id,
3618            turn_id,
3619            message,
3620            timestamp: OffsetDateTime::now_utc(),
3621        }))
3622        .await;
3623        Ok(())
3624    }
3625
3626    pub async fn tool_specs(&self) -> Vec<roder_api::tools::ToolSpec> {
3627        let cfg = self.config.read().await;
3628        let model_profile =
3629            model_profile_for_provider_model(&cfg, &cfg.default_provider, &cfg.default_model);
3630        self.filtered_tool_specs(&cfg, &cfg.default_model, model_profile.as_ref(), &[], &[])
3631    }
3632
3633    pub fn subagent_definitions(&self) -> Vec<SubagentDefinition> {
3634        self.registry
3635            .subagent_dispatchers
3636            .iter()
3637            .flat_map(|dispatcher| dispatcher.definitions())
3638            .collect()
3639    }
3640
3641    async fn run_turn(
3642        self: &Arc<Self>,
3643        req: StartTurnRequest,
3644        turn_id: TurnId,
3645        initial_mailbox_ack: Option<MailboxDeliveryAck>,
3646    ) -> anyhow::Result<TurnRunOutcome> {
3647        let turn_started_at = OffsetDateTime::now_utc();
3648        self.emit(RoderEvent::TurnStarted(TurnStarted {
3649            thread_id: req.thread_id.clone(),
3650            turn_id: turn_id.clone(),
3651            runtime_profile: self.config.read().await.runtime_profile,
3652            timestamp: turn_started_at,
3653        }))
3654        .await;
3655        self.persist_turn_item(
3656            &req.thread_id,
3657            &turn_id,
3658            &TranscriptItem::UserMessage(UserMessage::with_images(
3659                req.message.clone(),
3660                req.images.clone(),
3661            )),
3662        )
3663        .await?;
3664        if let Some(ack) = initial_mailbox_ack {
3665            self.teams
3666                .mark_mailbox_messages_delivered(&ack.team_id, &turn_id, &ack.message_ids)
3667                .await?;
3668        }
3669
3670        let mut cfg = self.config.read().await.clone();
3671        let runtime_profile = cfg.runtime_profile;
3672        let turn_deadline = turn_deadline_for_config(&cfg);
3673        let deadline_finalization_reserve =
3674            crate::deadline_policy::finalization_reserve_seconds(cfg.turn_deadline_seconds);
3675        let selection_mode = self.selection_mode_for_thread(&req.thread_id).await?;
3676        let concrete_selection = selection_mode
3677            .as_ref()
3678            .map(ModelSelectionMode::concrete_selection);
3679        let default_provider = req
3680            .provider_override
3681            .clone()
3682            .or_else(|| {
3683                concrete_selection
3684                    .as_ref()
3685                    .map(|selection| selection.provider.clone())
3686            })
3687            .unwrap_or(cfg.default_provider.clone());
3688        let default_model = req
3689            .model_override
3690            .clone()
3691            .or_else(|| {
3692                concrete_selection
3693                    .as_ref()
3694                    .map(|selection| selection.model.clone())
3695            })
3696            .unwrap_or(cfg.default_model.clone());
3697        if let Some(reasoning) = req.reasoning_override.as_deref().or_else(|| {
3698            selection_mode
3699                .as_ref()
3700                .and_then(ModelSelectionMode::reasoning)
3701        }) {
3702            validate_reasoning_effort(&default_model, reasoning)?;
3703            cfg.reasoning = Some(reasoning.to_string());
3704        }
3705        let turn_has_concrete_model_override =
3706            req.provider_override.is_some() || req.model_override.is_some();
3707        let (turn_inference_router, turn_inference_router_profile) = match &selection_mode {
3708            Some(ModelSelectionMode::Auto {
3709                router_id, profile, ..
3710            }) if !turn_has_concrete_model_override => (
3711                RuntimeInferenceRouterConfig {
3712                    enabled: true,
3713                    router_id: Some(router_id.clone()),
3714                },
3715                profile.clone(),
3716            ),
3717            _ => (RuntimeInferenceRouterConfig::disabled(), None),
3718        };
3719        let mut provider = default_provider.clone();
3720        let mut model = default_model.clone();
3721        let mut model_profile = model_profile_for_provider_model(&cfg, &provider, &model);
3722        let workspace = req.workspace.clone();
3723        let mut transcript = self.transcript_for_turn(&req, &turn_id, &model).await?;
3724        let mut compacted_this_turn = transcript
3725            .iter()
3726            .any(crate::compaction::is_compaction_boundary);
3727        let effective_policy_mode = self.effective_policy_mode_for_thread(&req.thread_id).await;
3728        let agent_swarm_mode_active = self
3729            .effective_agent_swarm_mode_for_thread(&req.thread_id)
3730            .await;
3731        let thread_overrides = self.thread_turn_overrides(&req.thread_id).await?;
3732        let mut final_assistant_text = String::new();
3733        let mut final_phase_messages = Vec::<AssistantMessage>::new();
3734        let mut final_reasoning_text = String::new();
3735        let mut final_provider_metadata = None;
3736        let mut exhausted_tool_rounds = true;
3737        let mut verification_gate =
3738            VerificationGateState::new(req.message.clone(), runtime_profile);
3739        let mut speed_policy = SpeedPolicyState::default();
3740        let mut reliability = TurnReliabilityState::default();
3741        let mut turn_usage = TokenUsage::default();
3742        // Overwritten on every inference step's Completed event so only the
3743        // terminal step's stop reason survives (mid-turn tool_use steps must
3744        // not leak onto turn/completed).
3745        let mut turn_finish_reason: Option<String> = None;
3746        let mut deadline_finalization_requested = false;
3747        let mut deadline_scoreable_completion_requested = false;
3748        let mut task_ledger_completion_reminders = 0_u8;
3749        let mut task_ledger_scoreable_checkpoints = 0_u8;
3750        let mut empty_tool_call_nudges_used = 0_u32;
3751        let mut provider_stream_retry_attempts = 0_u32;
3752        let mut routing_candidates = None;
3753        let routing_transcript_start = transcript.len().saturating_sub(1);
3754        let mut routing_escalations = 0_u32;
3755        let mut model_switch_summary_selection = None::<ModelSelection>;
3756
3757        'tool_rounds: for round_index in 0..MAX_TOOL_ROUNDS_PER_TURN {
3758            if let Some(deadline) = turn_deadline
3759                && deadline_expired(deadline)
3760            {
3761                self.fail_turn_due_to_deadline(&req.thread_id, &turn_id, deadline, &transcript)
3762                    .await?;
3763                return Ok(TurnRunOutcome::Stopped);
3764            }
3765            let steers = self.drain_turn_steers(&turn_id).await;
3766            self.append_steers(&req, &turn_id, &mut transcript, steers)
3767                .await?;
3768            if runtime_profile == RuntimeProfile::Eval
3769                && let Some(remaining) = crate::deadline_policy::should_start_finalization(
3770                    turn_deadline,
3771                    deadline_finalization_reserve,
3772                    deadline_finalization_requested || deadline_scoreable_completion_requested,
3773                )
3774            {
3775                if req.task_ledger_required
3776                    && task_ledger_completion_reminders < TASK_LEDGER_COMPLETION_REMINDER_LIMIT
3777                    && let Some(prompt) = task_ledger_completion_prompt(&transcript)
3778                {
3779                    task_ledger_completion_reminders += 1;
3780                    deadline_scoreable_completion_requested = true;
3781                    let item = TranscriptItem::UserMessage(UserMessage::text(
3782                        task_ledger_deadline_completion_prompt(
3783                            remaining,
3784                            deadline_finalization_reserve,
3785                            &prompt,
3786                        ),
3787                    ));
3788                    self.persist_turn_item(&req.thread_id, &turn_id, &item)
3789                        .await?;
3790                    transcript.push(item);
3791                    continue 'tool_rounds;
3792                } else {
3793                    self.start_deadline_finalization(
3794                        &req.thread_id,
3795                        &turn_id,
3796                        &mut transcript,
3797                        remaining,
3798                    )
3799                    .await?;
3800                    deadline_finalization_requested = true;
3801                }
3802            }
3803            if runtime_profile == RuntimeProfile::Eval
3804                && req.task_ledger_required
3805                && !deadline_finalization_requested
3806                && task_ledger_scoreable_checkpoints < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
3807                && let Some(remaining) = deadline_remaining_seconds(turn_deadline)
3808                && remaining <= TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS
3809                && remaining > deadline_finalization_reserve
3810                && let Some(prompt) = task_ledger_completion_prompt(&transcript)
3811            {
3812                task_ledger_scoreable_checkpoints += 1;
3813                let item = TranscriptItem::UserMessage(UserMessage::text(
3814                    task_ledger_scoreable_checkpoint_prompt(remaining, &prompt),
3815                ));
3816                self.persist_turn_item(&req.thread_id, &turn_id, &item)
3817                    .await?;
3818                transcript.push(item);
3819                continue 'tool_rounds;
3820            }
3821            if turn_inference_router.is_active() && routing_candidates.is_none() {
3822                routing_candidates =
3823                    Some(collect_inference_routing_candidates(&self.registry).await);
3824            }
3825            let routing_tools_model = model.clone();
3826            let routing_tools = self.filtered_tool_specs(
3827                &cfg,
3828                &model,
3829                model_profile.as_ref(),
3830                &thread_overrides.tool_allowlist,
3831                &thread_overrides.external_tools,
3832            );
3833            let prior_failures =
3834                transcript_failure_count_since(&transcript, routing_transcript_start)
3835                    .max(reliability.tool_failure_count())
3836                    .saturating_add(provider_stream_retry_attempts);
3837            let routing_selection = route_inference_selection(
3838                &self.registry,
3839                &turn_inference_router,
3840                InferenceRoutingRequest {
3841                    thread_id: &req.thread_id,
3842                    turn_id: &turn_id,
3843                    round_index: round_index as u32,
3844                    runtime_profile,
3845                    phase: speed_policy.phase(),
3846                    profile: turn_inference_router_profile.as_deref(),
3847                    default_selection: ModelSelection {
3848                        provider: default_provider.clone(),
3849                        model: default_model.clone(),
3850                    },
3851                    transcript: &transcript,
3852                    tools: &routing_tools,
3853                    candidates: routing_candidates.as_deref(),
3854                    prior_failures,
3855                    prior_escalations: routing_escalations,
3856                },
3857            )
3858            .await;
3859            if let Some(decision) = routing_selection.decision.clone() {
3860                if matches!(decision.outcome, InferenceRoutingOutcome::Escalated) {
3861                    routing_escalations = routing_escalations.saturating_add(1);
3862                }
3863                self.emit(RoderEvent::InferenceRoutingDecision(
3864                    InferenceRoutingDecisionEvent {
3865                        thread_id: req.thread_id.clone(),
3866                        turn_id: turn_id.clone(),
3867                        round_index: round_index as u32,
3868                        default_selection: ModelSelection {
3869                            provider: default_provider.clone(),
3870                            model: default_model.clone(),
3871                        },
3872                        selected_selection: routing_selection.selection.clone(),
3873                        decision,
3874                        timestamp: OffsetDateTime::now_utc(),
3875                    },
3876                ))
3877                .await;
3878            }
3879            provider = routing_selection.selection.provider.clone();
3880            model = routing_selection.selection.model.clone();
3881            let engine = self.engine_for(&provider)?;
3882            let capabilities = engine.capabilities();
3883            model_profile = model_profile_for_provider_model(&cfg, &provider, &model);
3884            let tools = if capabilities.tool_calls {
3885                if model == routing_tools_model {
3886                    routing_tools.clone()
3887                } else {
3888                    self.filtered_tool_specs(
3889                        &cfg,
3890                        &model,
3891                        model_profile.as_ref(),
3892                        &thread_overrides.tool_allowlist,
3893                        &thread_overrides.external_tools,
3894                    )
3895                }
3896            } else {
3897                Vec::new()
3898            };
3899            let parallel_tool_calls = parallel_tool_calls_for_model(&cfg, &model);
3900            let tool_choice = if tools.is_empty() {
3901                ToolChoice::None
3902            } else {
3903                ToolChoice::Auto
3904            };
3905            let summary_selection = ModelSelection {
3906                provider: provider.clone(),
3907                model: model.clone(),
3908            };
3909            if model_switch_summary_selection.as_ref() != Some(&summary_selection) {
3910                if let Some(summary) = model_switch_summary(
3911                    &transcript,
3912                    model_profile.as_ref(),
3913                    &provider,
3914                    &model,
3915                    &tools,
3916                ) {
3917                    let item = TranscriptItem::UserMessage(UserMessage::text(summary));
3918                    self.persist_turn_item(&req.thread_id, &turn_id, &item)
3919                        .await?;
3920                    transcript.push(item);
3921                }
3922                model_switch_summary_selection = Some(summary_selection);
3923            }
3924
3925            if !capabilities.image_input && transcript_has_images(&transcript) {
3926                self.fail_turn_with_error(
3927                    &req.thread_id,
3928                    &turn_id,
3929                    format!("provider {provider} does not support image input"),
3930                )
3931                .await?;
3932                return Ok(TurnRunOutcome::Stopped);
3933            }
3934            transcript = self
3935                .compact_transcript_if_needed(
3936                    &req.thread_id,
3937                    &turn_id,
3938                    &provider,
3939                    &model,
3940                    transcript,
3941                    self.compaction_options_for_turn(&req.thread_id, !compacted_this_turn),
3942                )
3943                .await?;
3944            compacted_this_turn = compacted_this_turn
3945                || transcript
3946                    .iter()
3947                    .any(crate::compaction::is_compaction_boundary);
3948
3949            let speed_policy_decision =
3950                speed_policy.decision(runtime_profile, &model, &cfg.speed_policy);
3951            let request_reasoning = reasoning_from_decision(
3952                speed_policy_decision.as_ref(),
3953                routing_selection
3954                    .reasoning
3955                    .clone()
3956                    .unwrap_or_else(|| reasoning_for_model(&cfg, &model)),
3957            );
3958            self.active_turn_selections.write().await.insert(
3959                turn_id.clone(),
3960                ModelSelectionMode::manual(
3961                    provider.clone(),
3962                    model.clone(),
3963                    request_reasoning.level.clone(),
3964                ),
3965            );
3966            if let Some(limit) = reliability.record_model_call(
3967                &cfg.reliability,
3968                runtime_profile == RuntimeProfile::Interactive,
3969            ) {
3970                self.fail_turn_due_to_reliability_limit(
3971                    &req.thread_id,
3972                    &turn_id,
3973                    &provider,
3974                    &model,
3975                    limit,
3976                    &transcript,
3977                )
3978                .await?;
3979                return Ok(TurnRunOutcome::Stopped);
3980            }
3981            self.emit(RoderEvent::InferenceStarted(InferenceStarted {
3982                thread_id: req.thread_id.clone(),
3983                turn_id: turn_id.clone(),
3984                engine_id: engine.id(),
3985                model: ModelSelection {
3986                    provider: provider.clone(),
3987                    model: model.clone(),
3988                },
3989                reasoning: request_reasoning.clone(),
3990                speed_policy: speed_policy_decision.clone(),
3991                deadline_remaining_seconds: deadline_remaining_seconds(turn_deadline),
3992                timestamp: OffsetDateTime::now_utc(),
3993            }))
3994            .await;
3995
3996            let mut instructions = req.instructions.clone();
3997            if let Some(extra) = &thread_overrides.developer_instructions {
3998                instructions = apply_thread_developer_instructions(instructions, extra);
3999            }
4000            if let Some(context) = req.developer_context.as_deref() {
4001                instructions = apply_turn_developer_context(instructions, context);
4002            }
4003            let mut instructions = apply_runtime_profile(instructions, runtime_profile);
4004            if let Some(profile) = &model_profile {
4005                instructions = apply_model_instruction_overlay(instructions, profile);
4006            }
4007            if req.task_ledger_required
4008                && runtime_profile == RuntimeProfile::Eval
4009                && !transcript_has_task_ledger(&transcript)
4010            {
4011                instructions = apply_task_ledger_required(instructions);
4012            }
4013            if effective_policy_mode == PolicyMode::Plan {
4014                instructions = apply_plan_mode(instructions);
4015            }
4016            if agent_swarm_mode_active {
4017                instructions = apply_agent_swarm_mode(instructions);
4018            }
4019            if model_supports_reasoning_effort(&model, REASONING_ULTRA) {
4020                instructions = apply_codex_multi_agent_mode(
4021                    instructions,
4022                    request_reasoning.level.as_deref() == Some(REASONING_ULTRA),
4023                );
4024            }
4025            instructions = self
4026                .goals
4027                .apply_goal_instructions(&req.thread_id, instructions)
4028                .await?;
4029            let mut request_metadata = serde_json::json!({});
4030            if let Some(decision) = &speed_policy_decision {
4031                request_metadata["speedPolicy"] = serde_json::json!(decision);
4032            }
4033            if let Some(decision) = routing_selection.decision.as_ref() {
4034                request_metadata["inferenceRouting"] = serde_json::json!(decision);
4035            }
4036            if let Some(remaining) = deadline_remaining_seconds(turn_deadline) {
4037                request_metadata["deadlineRemainingSeconds"] = serde_json::json!(remaining);
4038            }
4039            if let Some(profile) = &model_profile {
4040                request_metadata["modelProfile"] = serde_json::json!({
4041                    "model": profile.model,
4042                    "providerFamily": profile.provider_family,
4043                    "editTool": profile.edit_tool,
4044                    "schemaPolicy": profile.schema_policy,
4045                    "instructionOverlay": profile.instruction_overlay,
4046                    "parallelToolCalls": profile.parallel_tool_calls,
4047                    "autoCompactTokenLimit": profile.auto_compact_token_limit,
4048                });
4049            }
4050            let task_ledger_required_this_round = req.task_ledger_required
4051                && runtime_profile == RuntimeProfile::Eval
4052                && !deadline_finalization_requested
4053                && !transcript_has_task_ledger(&transcript);
4054            let task_ledger_tools = (capabilities.tool_calls && task_ledger_required_this_round)
4055                .then(|| self.task_ledger_tool_specs(model_profile.as_ref()))
4056                .filter(|tools| !tools.is_empty());
4057            let request_tools = if deadline_finalization_requested {
4058                Vec::new()
4059            } else if let Some(ledger_tools) = &task_ledger_tools {
4060                ledger_tools.clone()
4061            } else {
4062                tools.clone()
4063            };
4064            let request_tool_choice = if deadline_finalization_requested {
4065                ToolChoice::None
4066            } else if task_ledger_tools.is_some() {
4067                ToolChoice::Specific(TASK_LEDGER_TOOL_NAME.to_string())
4068            } else {
4069                tool_choice.clone()
4070            };
4071            if deadline_finalization_requested {
4072                request_metadata["deadlineFinalization"] = serde_json::json!({
4073                    "reserveSeconds": deadline_finalization_reserve,
4074                    "remainingSeconds": deadline_remaining_seconds(turn_deadline),
4075                });
4076            } else if deadline_scoreable_completion_requested {
4077                request_metadata["deadlineScoreableCompletion"] = serde_json::json!({
4078                    "reserveSeconds": deadline_finalization_reserve,
4079                    "remainingSeconds": deadline_remaining_seconds(turn_deadline),
4080                });
4081            }
4082            let request = AgentInferenceRequest {
4083                model: ModelSelection {
4084                    provider: provider.clone(),
4085                    model: model.clone(),
4086                },
4087                instructions,
4088                transcript: transcript.clone(),
4089                tools: request_tools,
4090                tool_choice: request_tool_choice,
4091                reasoning: request_reasoning,
4092                output: OutputConfig::default(),
4093                runtime: RuntimeHints {
4094                    auto_compact_token_limit: server_side_compaction_threshold(&cfg, &model),
4095                    profile: runtime_profile,
4096                    parallel_tool_calls: Some(parallel_tool_calls),
4097                    hosted_web_search: cfg.hosted_web_search.clone(),
4098                    tool_search: tool_search_for_provider_model(&cfg, &provider, &model),
4099                    speed_policy: speed_policy_decision,
4100                    reliability: Some(cfg.reliability.clone().into()),
4101                    deadline_remaining_seconds: deadline_remaining_seconds(turn_deadline),
4102                    ..RuntimeHints::default()
4103                },
4104                metadata: request_metadata,
4105            };
4106
4107            let ctx = InferenceTurnContext {
4108                thread_id: &req.thread_id,
4109                turn_id: &turn_id,
4110                tool_executor: Some(std::sync::Arc::new(
4111                    crate::tool_execution::RuntimeTurnToolExecutor {
4112                        runtime: Arc::clone(self),
4113                        thread_id: req.thread_id.clone(),
4114                        turn_id: turn_id.clone(),
4115                        workspace: Some(workspace.clone()),
4116                        deadline: turn_deadline,
4117                    },
4118                )),
4119            };
4120            let stream_future = engine.stream_turn(ctx, request);
4121            let mut stream = if let Some((deadline, timeout_action)) = inference_timeout_deadline(
4122                turn_deadline,
4123                runtime_profile,
4124                req.task_ledger_required,
4125                deadline_finalization_reserve,
4126                deadline_finalization_requested || deadline_scoreable_completion_requested,
4127                task_ledger_scoreable_checkpoints,
4128                &transcript,
4129            ) {
4130                match tokio::time::timeout_at(deadline_instant(deadline), stream_future).await {
4131                    Ok(stream) => stream?,
4132                    Err(_) => {
4133                        if runtime_profile == RuntimeProfile::Eval
4134                            && !deadline_finalization_requested
4135                        {
4136                            let remaining = deadline_remaining_seconds(turn_deadline).unwrap_or(0);
4137                            if timeout_action == InferenceTimeoutAction::ScoreableCheckpoint
4138                                && task_ledger_scoreable_checkpoints
4139                                    < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
4140                                && let Some(prompt) = task_ledger_completion_prompt(&transcript)
4141                            {
4142                                task_ledger_scoreable_checkpoints += 1;
4143                                let item = TranscriptItem::UserMessage(UserMessage::text(
4144                                    task_ledger_scoreable_checkpoint_prompt(remaining, &prompt),
4145                                ));
4146                                self.persist_turn_item(&req.thread_id, &turn_id, &item)
4147                                    .await?;
4148                                transcript.push(item);
4149                                continue 'tool_rounds;
4150                            }
4151                            self.start_deadline_finalization(
4152                                &req.thread_id,
4153                                &turn_id,
4154                                &mut transcript,
4155                                remaining,
4156                            )
4157                            .await?;
4158                            deadline_finalization_requested = true;
4159                            continue 'tool_rounds;
4160                        }
4161                        self.fail_turn_due_to_deadline(
4162                            &req.thread_id,
4163                            &turn_id,
4164                            deadline,
4165                            &transcript,
4166                        )
4167                        .await?;
4168                        return Ok(TurnRunOutcome::Stopped);
4169                    }
4170                }
4171            } else {
4172                stream_future.await?
4173            };
4174            let mut assistant_text = String::new();
4175            let mut phase_messages = Vec::<AssistantMessage>::new();
4176            let mut reasoning_text = String::new();
4177            let mut tool_calls = Vec::new();
4178            let mut provider_metadata = None;
4179            // Captured from mid-stream Compaction(completed) events. Codex/OpenAI
4180            // sometimes abort the SSE stream after emitting the compaction item
4181            // ("error decoding response body") before response.completed, so we
4182            // must not rely solely on ProviderMetadata for the boundary.
4183            let mut stream_compaction_item: Option<serde_json::Value> = None;
4184
4185            loop {
4186                let next = if let Some((deadline, timeout_action)) = inference_timeout_deadline(
4187                    turn_deadline,
4188                    runtime_profile,
4189                    req.task_ledger_required,
4190                    deadline_finalization_reserve,
4191                    deadline_finalization_requested || deadline_scoreable_completion_requested,
4192                    task_ledger_scoreable_checkpoints,
4193                    &transcript,
4194                ) {
4195                    match tokio::time::timeout_at(deadline_instant(deadline), stream.next()).await {
4196                        Ok(next) => next,
4197                        Err(_) => {
4198                            if runtime_profile == RuntimeProfile::Eval
4199                                && !deadline_finalization_requested
4200                            {
4201                                let remaining =
4202                                    deadline_remaining_seconds(turn_deadline).unwrap_or(0);
4203                                if timeout_action == InferenceTimeoutAction::ScoreableCheckpoint
4204                                    && task_ledger_scoreable_checkpoints
4205                                        < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
4206                                    && let Some(prompt) = task_ledger_completion_prompt(&transcript)
4207                                {
4208                                    task_ledger_scoreable_checkpoints += 1;
4209                                    let item = TranscriptItem::UserMessage(UserMessage::text(
4210                                        task_ledger_scoreable_checkpoint_prompt(remaining, &prompt),
4211                                    ));
4212                                    self.persist_turn_item(&req.thread_id, &turn_id, &item)
4213                                        .await?;
4214                                    transcript.push(item);
4215                                    continue 'tool_rounds;
4216                                }
4217                                self.start_deadline_finalization(
4218                                    &req.thread_id,
4219                                    &turn_id,
4220                                    &mut transcript,
4221                                    remaining,
4222                                )
4223                                .await?;
4224                                deadline_finalization_requested = true;
4225                                continue 'tool_rounds;
4226                            }
4227                            self.fail_turn_due_to_deadline(
4228                                &req.thread_id,
4229                                &turn_id,
4230                                deadline,
4231                                &transcript,
4232                            )
4233                            .await?;
4234                            return Ok(TurnRunOutcome::Stopped);
4235                        }
4236                    }
4237                } else {
4238                    stream.next().await
4239                };
4240                let Some(res) = next else {
4241                    break;
4242                };
4243                let event = match res {
4244                    Ok(event) => event,
4245                    Err(err) => {
4246                        let error = err.to_string();
4247                        if runtime_profile == RuntimeProfile::Eval
4248                            && !deadline_finalization_requested
4249                            && let Some(cause) = provider_stream_retry_cause(&error)
4250                        {
4251                            let retry_attempt = provider_stream_retry_attempts.saturating_add(1);
4252                            let policy: ReliabilityRequestPolicy = cfg.reliability.clone().into();
4253                            if retry_attempt < policy.provider_retry_max_attempts {
4254                                provider_stream_retry_attempts = retry_attempt;
4255                                let delay_ms = provider_retry_delay_ms(&policy, retry_attempt);
4256                                self.emit(RoderEvent::ReliabilityRetryRecorded(
4257                                    ReliabilityRetryRecorded {
4258                                        context: ReliabilityContext {
4259                                            thread_id: req.thread_id.clone(),
4260                                            turn_id: turn_id.clone(),
4261                                            provider: Some(provider.clone()),
4262                                            model: Some(model.clone()),
4263                                            ..ReliabilityContext::default()
4264                                        },
4265                                        error_class: ReliabilityErrorClass::ProviderError,
4266                                        decision: ReliabilityRetryDecision::Retry,
4267                                        attempt: retry_attempt,
4268                                        max_attempts: policy.provider_retry_max_attempts,
4269                                        delay_ms: Some(delay_ms),
4270                                        details: ReliabilityDetails::redacted(format!(
4271                                            "{cause}: {error}"
4272                                        )),
4273                                        timestamp: OffsetDateTime::now_utc(),
4274                                    },
4275                                ))
4276                                .await;
4277                                if delay_ms > 0 {
4278                                    tokio::time::sleep(std::time::Duration::from_millis(delay_ms))
4279                                        .await;
4280                                }
4281                                continue 'tool_rounds;
4282                            }
4283                        }
4284                        self.emit(RoderEvent::TurnFailed(TurnFailed {
4285                            thread_id: req.thread_id.clone(),
4286                            turn_id: turn_id.clone(),
4287                            error: error.clone(),
4288                            error_kind: None,
4289                            usage: None,
4290                            timestamp: OffsetDateTime::now_utc(),
4291                        }))
4292                        .await;
4293                        self.complete_team_member_turn_with_result(
4294                            &req.thread_id,
4295                            &turn_id,
4296                            TeamMemberStatus::Failed,
4297                            (!assistant_text.trim().is_empty()).then(|| assistant_text.clone()),
4298                            Some(error),
4299                        )
4300                        .await?;
4301                        return Err(err);
4302                    }
4303                };
4304
4305                let inference_timestamp = OffsetDateTime::now_utc();
4306                self.emit(RoderEvent::InferenceEventReceived(InferenceEventReceived {
4307                    thread_id: req.thread_id.clone(),
4308                    turn_id: turn_id.clone(),
4309                    event: event.clone(),
4310                    timestamp: inference_timestamp,
4311                }))
4312                .await;
4313
4314                match event {
4315                    InferenceEvent::MessageDelta(delta) => {
4316                        if let Some((team_id, member)) =
4317                            self.teams.member_for_thread(&req.thread_id).await
4318                        {
4319                            self.emit(RoderEvent::TeamMemberMessageDelta(TeamMemberMessageDelta {
4320                                team_id,
4321                                member_id: member.id,
4322                                member_thread_id: req.thread_id.clone(),
4323                                turn_id: turn_id.clone(),
4324                                delta: delta.text.clone(),
4325                                timestamp: OffsetDateTime::now_utc(),
4326                            }))
4327                            .await;
4328                        }
4329                        if is_final_answer_phase(delta.phase.as_deref()) {
4330                            assistant_text.push_str(&delta.text);
4331                        } else if let Some(last) = phase_messages.last_mut()
4332                            && last.phase == delta.phase
4333                        {
4334                            last.text.push_str(&delta.text);
4335                        } else {
4336                            phase_messages.push(AssistantMessage {
4337                                text: delta.text,
4338                                phase: delta.phase,
4339                            });
4340                        }
4341                    }
4342                    InferenceEvent::ReasoningDelta(delta) => reasoning_text.push_str(&delta.text),
4343                    InferenceEvent::ToolCallCompleted(call) => tool_calls.push(call),
4344                    InferenceEvent::Failed(failure) => {
4345                        speed_policy.record_failure();
4346                        let error = failure.message;
4347                        self.persist_turn_item(
4348                            &req.thread_id,
4349                            &turn_id,
4350                            &TranscriptItem::Error(ErrorRecord {
4351                                message: error.clone(),
4352                            }),
4353                        )
4354                        .await?;
4355                        self.emit(RoderEvent::TurnFailed(TurnFailed {
4356                            thread_id: req.thread_id.clone(),
4357                            turn_id: turn_id.clone(),
4358                            error: error.clone(),
4359                            error_kind: None,
4360                            usage: None,
4361                            timestamp: OffsetDateTime::now_utc(),
4362                        }))
4363                        .await;
4364                        self.complete_team_member_turn_with_result(
4365                            &req.thread_id,
4366                            &turn_id,
4367                            TeamMemberStatus::Failed,
4368                            (!assistant_text.trim().is_empty()).then(|| assistant_text.clone()),
4369                            Some(error),
4370                        )
4371                        .await?;
4372                        return Ok(TurnRunOutcome::Stopped);
4373                    }
4374                    InferenceEvent::Usage(usage) => {
4375                        turn_usage.add_assign(&usage);
4376                    }
4377                    InferenceEvent::Completed(metadata) => {
4378                        turn_finish_reason = metadata
4379                            .stop_reason
4380                            .as_deref()
4381                            .map(finish_reason_from_stop_reason);
4382                    }
4383                    InferenceEvent::Compaction(progress) => {
4384                        // Persist the boundary as soon as the provider emits a
4385                        // completed compaction item. ChatGPT Codex often aborts
4386                        // the SSE stream right after ("error decoding response
4387                        // body"), so waiting for ProviderMetadata loses the
4388                        // boundary and every subsequent request re-compacts.
4389                        if progress.status == "completed"
4390                            && let Some(item) = progress.item
4391                        {
4392                            let already = stream_compaction_item
4393                                .as_ref()
4394                                .and_then(|existing| {
4395                                    existing.get("id").and_then(serde_json::Value::as_str)
4396                                })
4397                                .zip(item.get("id").and_then(serde_json::Value::as_str))
4398                                .is_some_and(|(a, b)| a == b);
4399                            if !already {
4400                                stream_compaction_item = Some(item.clone());
4401                                let boundary = TranscriptItem::ProviderMetadata(
4402                                    crate::compaction::provider_metadata_from_compaction_item(item),
4403                                );
4404                                self.persist_turn_item(&req.thread_id, &turn_id, &boundary)
4405                                    .await?;
4406                                transcript.push(boundary);
4407                                transcript =
4408                                    crate::compaction::trim_to_last_compaction_boundary(transcript);
4409                                compacted_this_turn = true;
4410                            }
4411                        }
4412                    }
4413                    InferenceEvent::HostedToolCallStarted(_)
4414                    | InferenceEvent::HostedToolCallCompleted(_)
4415                    | InferenceEvent::ToolCallStarted(_)
4416                    | InferenceEvent::ToolCallDelta(_) => {}
4417                    InferenceEvent::ProviderMetadata(mut metadata) => {
4418                        if let Some(compaction_item) = stream_compaction_item.as_ref() {
4419                            let _ = crate::compaction::ensure_provider_metadata_compaction(
4420                                &mut metadata,
4421                                compaction_item,
4422                            );
4423                        }
4424                        provider_metadata = Some(metadata);
4425                    }
4426                }
4427            }
4428
4429            speed_policy.record_model_output(
4430                !assistant_text.is_empty() || !phase_messages.is_empty(),
4431                tool_calls.len(),
4432            );
4433            if tool_calls.is_empty() {
4434                let steers = self.drain_turn_steers(&turn_id).await;
4435                if !steers.is_empty() {
4436                    for message in phase_messages {
4437                        let item = TranscriptItem::AssistantMessage(message);
4438                        self.persist_turn_item(&req.thread_id, &turn_id, &item)
4439                            .await?;
4440                        transcript.push(item);
4441                        self.persist_model_profile_segment(
4442                            &req.thread_id,
4443                            &turn_id,
4444                            model_profile.as_ref(),
4445                            &provider,
4446                            &model,
4447                            "assistant",
4448                        )
4449                        .await?;
4450                    }
4451                    if !assistant_text.is_empty() {
4452                        let assistant = TranscriptItem::AssistantMessage(AssistantMessage {
4453                            text: assistant_text,
4454                            phase: Some(FINAL_ANSWER_PHASE.to_string()),
4455                        });
4456                        self.persist_turn_item(&req.thread_id, &turn_id, &assistant)
4457                            .await?;
4458                        transcript.push(assistant);
4459                        self.persist_model_profile_segment(
4460                            &req.thread_id,
4461                            &turn_id,
4462                            model_profile.as_ref(),
4463                            &provider,
4464                            &model,
4465                            "assistant",
4466                        )
4467                        .await?;
4468                    }
4469                    if let Some(metadata) = provider_metadata {
4470                        let had_provider_compaction =
4471                            crate::compaction::provider_metadata_has_compaction(&metadata);
4472                        let item = TranscriptItem::ProviderMetadata(metadata);
4473                        self.persist_turn_item(&req.thread_id, &turn_id, &item)
4474                            .await?;
4475                        transcript.push(item);
4476                        if had_provider_compaction {
4477                            // Server-side compaction replaced the prior window;
4478                            // drop pre-boundary items so the next request does
4479                            // not re-send (and re-compact) the full history.
4480                            transcript =
4481                                crate::compaction::trim_to_last_compaction_boundary(transcript);
4482                            compacted_this_turn = true;
4483                        }
4484                    }
4485                    self.append_steers(&req, &turn_id, &mut transcript, steers)
4486                        .await?;
4487                    continue;
4488                }
4489                if !deadline_finalization_requested
4490                    && req.task_ledger_required
4491                    && runtime_profile == RuntimeProfile::Eval
4492                    && task_ledger_completion_reminders < TASK_LEDGER_COMPLETION_REMINDER_LIMIT
4493                    && (!assistant_text.trim().is_empty() || !phase_messages.is_empty())
4494                    && let Some(prompt) = task_ledger_completion_prompt(&transcript)
4495                {
4496                    task_ledger_completion_reminders += 1;
4497                    let item = TranscriptItem::UserMessage(UserMessage::text(prompt));
4498                    self.persist_turn_item(&req.thread_id, &turn_id, &item)
4499                        .await?;
4500                    transcript.push(item);
4501                    continue;
4502                }
4503                if !deadline_finalization_requested
4504                    && let Some(prompt) = verification_gate.blocking_prompt()
4505                {
4506                    speed_policy.record_verification_required();
4507                    self.emit(RoderEvent::VerificationRequired(VerificationRequired {
4508                        thread_id: req.thread_id.clone(),
4509                        turn_id: turn_id.clone(),
4510                        reason: verification_gate.reason(),
4511                        changed_files: verification_gate.changed_files(),
4512                        tool_evidence: verification_gate.tool_evidence.clone(),
4513                        tests_run: verification_gate.tests_run.clone(),
4514                        open_gaps: verification_gate.open_gaps.clone(),
4515                        timestamp: OffsetDateTime::now_utc(),
4516                    }))
4517                    .await;
4518                    let item = TranscriptItem::UserMessage(UserMessage::text(prompt));
4519                    self.persist_turn_item(&req.thread_id, &turn_id, &item)
4520                        .await?;
4521                    transcript.push(item);
4522                    continue;
4523                }
4524                // Loop persistence nudge: in non-interactive/eval runs, when the
4525                // model returns a final message with no tool calls, gently prompt
4526                // it to keep going (bounded by `empty_tool_call_nudges`) before
4527                // ending the turn. Gated to non-interactive/eval so interactive
4528                // chat completions are never delayed, and only fires when the
4529                // model actually produced a final answer to re-examine.
4530                if !deadline_finalization_requested
4531                    && runtime_profile != RuntimeProfile::Interactive
4532                    && cfg.reliability.empty_tool_call_nudges > 0
4533                    && empty_tool_call_nudges_used < cfg.reliability.empty_tool_call_nudges
4534                    && (!assistant_text.trim().is_empty() || !phase_messages.is_empty())
4535                {
4536                    empty_tool_call_nudges_used += 1;
4537                    let item = TranscriptItem::UserMessage(UserMessage::text(
4538                        RELIABILITY_CONTINUATION_PROMPT.to_string(),
4539                    ));
4540                    self.persist_turn_item(&req.thread_id, &turn_id, &item)
4541                        .await?;
4542                    transcript.push(item);
4543                    continue;
4544                }
4545                if deadline_finalization_requested
4546                    && assistant_text.trim().is_empty()
4547                    && phase_messages.is_empty()
4548                {
4549                    assistant_text = format!(
4550                        "Deadline finalization completed without model text. {}",
4551                        turn_partial_result(&transcript)
4552                    );
4553                }
4554                final_phase_messages = phase_messages;
4555                final_assistant_text = assistant_text;
4556                final_reasoning_text = reasoning_text;
4557                final_provider_metadata = provider_metadata;
4558                exhausted_tool_rounds = false;
4559                break;
4560            }
4561
4562            for message in phase_messages {
4563                let item = TranscriptItem::AssistantMessage(message);
4564                self.persist_turn_item(&req.thread_id, &turn_id, &item)
4565                    .await?;
4566                transcript.push(item);
4567                self.persist_model_profile_segment(
4568                    &req.thread_id,
4569                    &turn_id,
4570                    model_profile.as_ref(),
4571                    &provider,
4572                    &model,
4573                    "assistant",
4574                )
4575                .await?;
4576            }
4577            if !assistant_text.is_empty() {
4578                transcript.push(TranscriptItem::AssistantMessage(AssistantMessage {
4579                    text: assistant_text,
4580                    phase: Some(FINAL_ANSWER_PHASE.to_string()),
4581                }));
4582                self.persist_model_profile_segment(
4583                    &req.thread_id,
4584                    &turn_id,
4585                    model_profile.as_ref(),
4586                    &provider,
4587                    &model,
4588                    "assistant",
4589                )
4590                .await?;
4591            }
4592            if let Some(metadata) = provider_metadata {
4593                let had_provider_compaction =
4594                    crate::compaction::provider_metadata_has_compaction(&metadata);
4595                let item = TranscriptItem::ProviderMetadata(metadata);
4596                self.persist_turn_item(&req.thread_id, &turn_id, &item)
4597                    .await?;
4598                transcript.push(item);
4599                if had_provider_compaction {
4600                    // Server-side compaction replaced the prior window; drop
4601                    // pre-boundary items so subsequent tool rounds use the
4602                    // compact window as the next input.
4603                    transcript = crate::compaction::trim_to_last_compaction_boundary(transcript);
4604                    compacted_this_turn = true;
4605                }
4606            }
4607            for call in &tool_calls {
4608                let tool_item = TranscriptItem::ToolCall(ToolCallRecord {
4609                    id: call.id.clone(),
4610                    name: call.name.clone(),
4611                    arguments: call.arguments.clone(),
4612                });
4613                self.persist_turn_item(&req.thread_id, &turn_id, &tool_item)
4614                    .await?;
4615                transcript.push(tool_item);
4616                self.persist_model_profile_segment(
4617                    &req.thread_id,
4618                    &turn_id,
4619                    model_profile.as_ref(),
4620                    &provider,
4621                    &model,
4622                    "tool_call",
4623                )
4624                .await?;
4625            }
4626            if let Some(deadline) = turn_deadline
4627                && deadline_expired(deadline)
4628            {
4629                self.fail_turn_due_to_deadline(&req.thread_id, &turn_id, deadline, &transcript)
4630                    .await?;
4631                return Ok(TurnRunOutcome::Stopped);
4632            }
4633            let results = self
4634                .route_tool_calls(
4635                    &req.thread_id,
4636                    &turn_id,
4637                    tool_calls,
4638                    parallel_tool_calls,
4639                    Some(workspace.as_str()),
4640                    turn_deadline,
4641                )
4642                .await?;
4643            let reliability_limit = reliability.record_tool_results(
4644                &cfg.reliability,
4645                &results,
4646                runtime_profile == RuntimeProfile::Interactive,
4647            );
4648            for result in results {
4649                verification_gate.record_tool_result(&result);
4650                transcript.push(TranscriptItem::ToolResult(result));
4651                self.persist_model_profile_segment(
4652                    &req.thread_id,
4653                    &turn_id,
4654                    model_profile.as_ref(),
4655                    &provider,
4656                    &model,
4657                    "tool_result",
4658                )
4659                .await?;
4660            }
4661            if let Some(limit) = reliability_limit {
4662                if limit.decision == ReliabilityLimitDecision::RequestContinuation {
4663                    // Loop persistence: rather than ending the turn, reset the
4664                    // consecutive-failure counter, nudge the model to keep going,
4665                    // and continue the round loop. Bounded by the per-turn tool
4666                    // failure ceiling, `max_model_calls_per_turn`, and
4667                    // `MAX_TOOL_ROUNDS_PER_TURN`.
4668                    self.record_reliability_limit_continuation(
4669                        &req.thread_id,
4670                        &turn_id,
4671                        &provider,
4672                        &model,
4673                        limit,
4674                    )
4675                    .await;
4676                    reliability.reset_consecutive_failures();
4677                    let nudge = TranscriptItem::UserMessage(UserMessage::text(
4678                        RELIABILITY_CONTINUATION_PROMPT.to_string(),
4679                    ));
4680                    self.persist_turn_item(&req.thread_id, &turn_id, &nudge)
4681                        .await?;
4682                    transcript.push(nudge);
4683                } else {
4684                    self.fail_turn_due_to_reliability_limit(
4685                        &req.thread_id,
4686                        &turn_id,
4687                        &provider,
4688                        &model,
4689                        limit,
4690                        &transcript,
4691                    )
4692                    .await?;
4693                    return Ok(TurnRunOutcome::Stopped);
4694                }
4695            }
4696            transcript = self
4697                .compact_transcript_if_needed(
4698                    &req.thread_id,
4699                    &turn_id,
4700                    &provider,
4701                    &model,
4702                    transcript,
4703                    self.compaction_options_for_turn(&req.thread_id, !compacted_this_turn),
4704                )
4705                .await?;
4706            compacted_this_turn = compacted_this_turn
4707                || transcript
4708                    .iter()
4709                    .any(crate::compaction::is_compaction_boundary);
4710        }
4711
4712        if exhausted_tool_rounds {
4713            let message =
4714                format!("tool call limit reached after {MAX_TOOL_ROUNDS_PER_TURN} rounds");
4715            self.persist_turn_item(
4716                &req.thread_id,
4717                &turn_id,
4718                &TranscriptItem::Error(ErrorRecord {
4719                    message: message.clone(),
4720                }),
4721            )
4722            .await?;
4723            self.emit(RoderEvent::TurnFailed(TurnFailed {
4724                thread_id: req.thread_id.clone(),
4725                turn_id: turn_id.clone(),
4726                error: message.clone(),
4727                error_kind: None,
4728                usage: None,
4729                timestamp: OffsetDateTime::now_utc(),
4730            }))
4731            .await;
4732            self.complete_team_member_turn_with_result(
4733                &req.thread_id,
4734                &turn_id,
4735                TeamMemberStatus::Failed,
4736                None,
4737                Some(message),
4738            )
4739            .await?;
4740            return Ok(TurnRunOutcome::Stopped);
4741        }
4742
4743        if !final_reasoning_text.is_empty() {
4744            self.persist_turn_item(
4745                &req.thread_id,
4746                &turn_id,
4747                &TranscriptItem::ReasoningSummary(ReasoningSummary {
4748                    text: final_reasoning_text,
4749                }),
4750            )
4751            .await?;
4752        }
4753        for message in final_phase_messages {
4754            self.persist_turn_item(
4755                &req.thread_id,
4756                &turn_id,
4757                &TranscriptItem::AssistantMessage(message),
4758            )
4759            .await?;
4760            self.persist_model_profile_segment(
4761                &req.thread_id,
4762                &turn_id,
4763                model_profile.as_ref(),
4764                &provider,
4765                &model,
4766                "assistant",
4767            )
4768            .await?;
4769        }
4770        if !final_assistant_text.is_empty() {
4771            self.persist_turn_item(
4772                &req.thread_id,
4773                &turn_id,
4774                &TranscriptItem::AssistantMessage(AssistantMessage {
4775                    text: final_assistant_text.clone(),
4776                    phase: Some(FINAL_ANSWER_PHASE.to_string()),
4777                }),
4778            )
4779            .await?;
4780            self.persist_model_profile_segment(
4781                &req.thread_id,
4782                &turn_id,
4783                model_profile.as_ref(),
4784                &provider,
4785                &model,
4786                "assistant",
4787            )
4788            .await?;
4789        }
4790        if let Some(metadata) = final_provider_metadata {
4791            self.persist_turn_item(
4792                &req.thread_id,
4793                &turn_id,
4794                &TranscriptItem::ProviderMetadata(metadata),
4795            )
4796            .await?;
4797        }
4798
4799        let turn_usage_tokens = turn_usage.total_tokens as i64;
4800        let completed_usage = (!turn_usage.is_empty()).then_some(turn_usage.clone());
4801        self.record_thread_usage_metadata(&req.thread_id, &turn_usage)
4802            .await?;
4803        self.goals
4804            .account_turn_usage(
4805                &req.thread_id,
4806                turn_usage_tokens,
4807                OffsetDateTime::now_utc() - turn_started_at,
4808            )
4809            .await?;
4810        let (cleanup, ownership) = self.await_provider_turn_cleanup(&turn_id).await;
4811        self.record_turn_lifecycle_with_ownership(
4812            req.thread_id.clone(),
4813            turn_id.clone(),
4814            TurnLifecycleState::Completed,
4815            cleanup,
4816            None,
4817            ownership,
4818        )
4819        .await;
4820        self.emit(RoderEvent::TurnCompleted(TurnCompleted {
4821            thread_id: req.thread_id.clone(),
4822            turn_id: turn_id.clone(),
4823            usage: completed_usage,
4824            finish_reason: turn_finish_reason,
4825            timestamp: OffsetDateTime::now_utc(),
4826        }))
4827        .await;
4828        self.complete_team_member_turn_with_result(
4829            &req.thread_id,
4830            &turn_id,
4831            TeamMemberStatus::Completed,
4832            (!final_assistant_text.is_empty()).then_some(final_assistant_text),
4833            None,
4834        )
4835        .await?;
4836        Ok(TurnRunOutcome::Completed)
4837    }
4838
4839    async fn drain_turn_steers(&self, turn_id: &TurnId) -> Vec<QueuedTurnSteer> {
4840        let Some(active) = self.active_turns.read().await.get(turn_id).cloned() else {
4841            return Vec::new();
4842        };
4843        let mut steers = active.steers.lock().await;
4844        std::mem::take(&mut *steers)
4845    }
4846
4847    async fn route_tool_calls(
4848        self: &Arc<Self>,
4849        thread_id: &ThreadId,
4850        turn_id: &TurnId,
4851        calls: Vec<ToolCallCompleted>,
4852        parallel: bool,
4853        workspace: Option<&str>,
4854        deadline: Option<OffsetDateTime>,
4855    ) -> anyhow::Result<Vec<ToolResultRecord>> {
4856        // Enforce the agent_swarm exclusivity rule (roadmap 104, Task 2):
4857        // `agent_swarm` must be the only tool call in a model response. A mixed
4858        // or multi-swarm batch is denied wholesale with actionable retry text so
4859        // the model re-issues `agent_swarm` by itself, and every tool_call_id
4860        // still gets a response (keeping the chat-completions transcript valid).
4861        if let Some(violation) = roder_api::subagents::agent_swarm_batch_violation(
4862            calls.iter().map(|call| call.name.as_str()),
4863        ) {
4864            let message = violation.deny_message();
4865            return Ok(calls
4866                .into_iter()
4867                .map(|call| ToolResultRecord {
4868                    id: call.id,
4869                    name: Some(call.name),
4870                    result: message.clone(),
4871                    display_payload: None,
4872                    is_error: true,
4873                })
4874                .collect());
4875        }
4876        // Emit swarm lifecycle events on the bus (roadmap 104, Task 1) so any
4877        // app-server/SDK/TUI client can observe a swarm as a whole; per-child
4878        // progress flows through the existing Subagent* trace events.
4879        let swarm_call = (calls.len() == 1
4880            && calls[0].name == roder_api::subagents::AGENT_SWARM_TOOL_NAME)
4881            .then(|| (calls[0].id.clone(), calls[0].arguments.clone()));
4882        if let Some((tool_id, args)) = &swarm_call {
4883            self.emit(RoderEvent::AgentSwarmStarted(
4884                roder_api::subagents::AgentSwarmStarted {
4885                    thread_id: thread_id.clone(),
4886                    turn_id: turn_id.clone(),
4887                    tool_id: tool_id.clone(),
4888                    child_count: agent_swarm_child_count(args),
4889                    timestamp: OffsetDateTime::now_utc(),
4890                },
4891            ))
4892            .await;
4893        }
4894
4895        let force_sequential = calls
4896            .iter()
4897            .any(|call| crate::agent_control_tools::is_agent_control_tool(&call.name));
4898        let results =
4899            if parallel && !force_sequential {
4900                try_join_all(calls.into_iter().map(|call| {
4901                    self.route_tool_call(thread_id, turn_id, call, workspace, deadline)
4902                }))
4903                .await
4904            } else {
4905                let mut results = Vec::with_capacity(calls.len());
4906                for call in calls {
4907                    results.push(
4908                        self.route_tool_call(thread_id, turn_id, call, workspace, deadline)
4909                            .await?,
4910                    );
4911                }
4912                Ok(results)
4913            }?;
4914
4915        if let Some((tool_id, _)) = &swarm_call
4916            && let Some(result) = results.iter().find(|result| &result.id == tool_id)
4917            && let Some((completed, failed, aborted)) = parse_swarm_counts(&result.result)
4918        {
4919            self.emit(RoderEvent::AgentSwarmCompleted(
4920                roder_api::subagents::AgentSwarmCompleted {
4921                    thread_id: thread_id.clone(),
4922                    turn_id: turn_id.clone(),
4923                    tool_id: tool_id.clone(),
4924                    completed,
4925                    failed,
4926                    aborted,
4927                    timestamp: OffsetDateTime::now_utc(),
4928                },
4929            ))
4930            .await;
4931        }
4932
4933        Ok(results)
4934    }
4935
4936    async fn fail_turn_with_error(
4937        &self,
4938        thread_id: &ThreadId,
4939        turn_id: &TurnId,
4940        message: String,
4941    ) -> anyhow::Result<()> {
4942        self.persist_turn_item(
4943            thread_id,
4944            turn_id,
4945            &TranscriptItem::Error(ErrorRecord {
4946                message: message.clone(),
4947            }),
4948        )
4949        .await?;
4950        self.emit(RoderEvent::TurnFailed(TurnFailed {
4951            thread_id: thread_id.clone(),
4952            turn_id: turn_id.clone(),
4953            error: message.clone(),
4954            error_kind: None,
4955            usage: None,
4956            timestamp: OffsetDateTime::now_utc(),
4957        }))
4958        .await;
4959        self.complete_team_member_turn_with_result(
4960            thread_id,
4961            turn_id,
4962            TeamMemberStatus::Failed,
4963            None,
4964            Some(message),
4965        )
4966        .await?;
4967        Ok(())
4968    }
4969
4970    async fn fail_turn_due_to_deadline(
4971        &self,
4972        thread_id: &ThreadId,
4973        turn_id: &TurnId,
4974        deadline: OffsetDateTime,
4975        transcript: &[TranscriptItem],
4976    ) -> anyhow::Result<()> {
4977        let partial_result = turn_partial_result(transcript);
4978        self.emit(RoderEvent::TurnPartialResult(TurnPartialResult {
4979            thread_id: thread_id.clone(),
4980            turn_id: turn_id.clone(),
4981            summary: partial_result.clone(),
4982            timestamp: OffsetDateTime::now_utc(),
4983        }))
4984        .await;
4985        self.emit(RoderEvent::TurnDeadlineExceeded(TurnDeadlineExceeded {
4986            thread_id: thread_id.clone(),
4987            turn_id: turn_id.clone(),
4988            deadline,
4989            partial_result: partial_result.clone(),
4990            timestamp: OffsetDateTime::now_utc(),
4991        }))
4992        .await;
4993        let message = "turn deadline expired".to_string();
4994        self.persist_turn_item(
4995            thread_id,
4996            turn_id,
4997            &TranscriptItem::Error(ErrorRecord {
4998                message: format!("{message}: {partial_result}"),
4999            }),
5000        )
5001        .await?;
5002        self.emit(RoderEvent::TurnFailed(TurnFailed {
5003            thread_id: thread_id.clone(),
5004            turn_id: turn_id.clone(),
5005            error: message.clone(),
5006            error_kind: Some("deadline_timeout".to_string()),
5007            usage: None,
5008            timestamp: OffsetDateTime::now_utc(),
5009        }))
5010        .await;
5011        self.complete_team_member_turn_with_result(
5012            thread_id,
5013            turn_id,
5014            TeamMemberStatus::Failed,
5015            None,
5016            Some(format!("{message}: {partial_result}")),
5017        )
5018        .await?;
5019        Ok(())
5020    }
5021
5022    async fn start_deadline_finalization(
5023        &self,
5024        thread_id: &ThreadId,
5025        turn_id: &TurnId,
5026        transcript: &mut Vec<TranscriptItem>,
5027        remaining_seconds: u64,
5028    ) -> anyhow::Result<()> {
5029        let item = TranscriptItem::UserMessage(crate::deadline_policy::finalization_message(
5030            remaining_seconds,
5031        ));
5032        self.persist_turn_item(thread_id, turn_id, &item).await?;
5033        transcript.push(item);
5034        self.emit(RoderEvent::TurnPartialResult(TurnPartialResult {
5035            thread_id: thread_id.clone(),
5036            turn_id: turn_id.clone(),
5037            summary: turn_partial_result(transcript),
5038            timestamp: OffsetDateTime::now_utc(),
5039        }))
5040        .await;
5041        Ok(())
5042    }
5043
5044    /// Emits the reliability-limit event for a continuation (loop persistence)
5045    /// without failing the turn. The caller resets the failure counter and
5046    /// injects a nudge so the round loop keeps going.
5047    async fn record_reliability_limit_continuation(
5048        &self,
5049        thread_id: &ThreadId,
5050        turn_id: &TurnId,
5051        provider: &str,
5052        model: &str,
5053        limit: ReliabilityLimitHit,
5054    ) {
5055        self.emit(RoderEvent::ReliabilityLimitRecorded(
5056            ReliabilityLimitRecorded {
5057                context: ReliabilityContext {
5058                    thread_id: thread_id.clone(),
5059                    turn_id: turn_id.clone(),
5060                    tool_id: None,
5061                    tool_name: None,
5062                    provider: Some(provider.to_string()),
5063                    model: Some(model.to_string()),
5064                },
5065                error_class: limit.error_class,
5066                limit_kind: limit.limit_kind,
5067                decision: limit.decision,
5068                current: limit.current,
5069                limit: limit.limit,
5070                details: ReliabilityDetails::redacted(&limit.message),
5071                timestamp: OffsetDateTime::now_utc(),
5072            },
5073        ))
5074        .await;
5075    }
5076
5077    async fn fail_turn_due_to_reliability_limit(
5078        &self,
5079        thread_id: &ThreadId,
5080        turn_id: &TurnId,
5081        provider: &str,
5082        model: &str,
5083        limit: ReliabilityLimitHit,
5084        transcript: &[TranscriptItem],
5085    ) -> anyhow::Result<()> {
5086        self.emit(RoderEvent::ReliabilityLimitRecorded(
5087            ReliabilityLimitRecorded {
5088                context: ReliabilityContext {
5089                    thread_id: thread_id.clone(),
5090                    turn_id: turn_id.clone(),
5091                    tool_id: None,
5092                    tool_name: None,
5093                    provider: Some(provider.to_string()),
5094                    model: Some(model.to_string()),
5095                },
5096                error_class: limit.error_class,
5097                limit_kind: limit.limit_kind,
5098                decision: limit.decision,
5099                current: limit.current,
5100                limit: limit.limit,
5101                details: ReliabilityDetails::redacted(&limit.message),
5102                timestamp: OffsetDateTime::now_utc(),
5103            },
5104        ))
5105        .await;
5106        let partial_result = turn_partial_result(transcript);
5107        self.emit(RoderEvent::TurnPartialResult(TurnPartialResult {
5108            thread_id: thread_id.clone(),
5109            turn_id: turn_id.clone(),
5110            summary: partial_result.clone(),
5111            timestamp: OffsetDateTime::now_utc(),
5112        }))
5113        .await;
5114        let message = format!("reliability limit reached: {}", limit.message);
5115        self.persist_turn_item(
5116            thread_id,
5117            turn_id,
5118            &TranscriptItem::Error(ErrorRecord {
5119                message: format!("{message}: {partial_result}"),
5120            }),
5121        )
5122        .await?;
5123        self.emit(RoderEvent::TurnFailed(TurnFailed {
5124            thread_id: thread_id.clone(),
5125            turn_id: turn_id.clone(),
5126            error: message.clone(),
5127            error_kind: Some("reliability_limit".to_string()),
5128            usage: None,
5129            timestamp: OffsetDateTime::now_utc(),
5130        }))
5131        .await;
5132        self.complete_team_member_turn_with_result(
5133            thread_id,
5134            turn_id,
5135            TeamMemberStatus::Failed,
5136            None,
5137            Some(format!("{message}: {partial_result}")),
5138        )
5139        .await?;
5140        Ok(())
5141    }
5142
5143    async fn append_steers(
5144        &self,
5145        req: &StartTurnRequest,
5146        turn_id: &TurnId,
5147        transcript: &mut Vec<TranscriptItem>,
5148        steers: Vec<QueuedTurnSteer>,
5149    ) -> anyhow::Result<()> {
5150        for queued in steers {
5151            let mut steer = queued.message;
5152            steer.text = steer.text.trim().to_string();
5153            if steer.text.is_empty() && steer.images.is_empty() {
5154                continue;
5155            }
5156            let item = TranscriptItem::UserMessage(steer);
5157            self.persist_turn_item(&req.thread_id, turn_id, &item)
5158                .await?;
5159            transcript.push(item);
5160            if let Some(ack) = queued.mailbox_ack {
5161                self.teams
5162                    .mark_mailbox_messages_delivered(&ack.team_id, turn_id, &ack.message_ids)
5163                    .await?;
5164            }
5165        }
5166        Ok(())
5167    }
5168
5169    async fn persist_model_profile_segment(
5170        &self,
5171        thread_id: &ThreadId,
5172        turn_id: &TurnId,
5173        profile: Option<&ModelHarnessProfile>,
5174        provider: &str,
5175        model: &str,
5176        segment: &str,
5177    ) -> anyhow::Result<()> {
5178        let item = TranscriptItem::ProviderMetadata(model_profile_segment_metadata(
5179            profile, provider, model, segment,
5180        ));
5181        self.persist_turn_item(thread_id, turn_id, &item).await
5182    }
5183
5184    /**
5185     * Allowlists apply to built-in tools only; external tools are advertised with their
5186     * host-supplied schemas as given. An external tool shadows a built-in with the same name in
5187     * both advertisement and dispatch (see `route_tool_call`).
5188     */
5189    fn filtered_tool_specs(
5190        &self,
5191        cfg: &RuntimeConfig,
5192        model: &str,
5193        profile: Option<&ModelHarnessProfile>,
5194        thread_allowlist: &[String],
5195        external_tools: &[roder_api::tools::ToolSpec],
5196    ) -> Vec<roder_api::tools::ToolSpec> {
5197        let mut specs = self
5198            .tool_registry
5199            .specs_for_edit_tool_with_schema_policy(
5200                edit_tool_for_model(cfg, model),
5201                schema_policy_for_model(profile),
5202            )
5203            .into_iter()
5204            .filter(|spec| {
5205                allowlist_permits(&cfg.tool_allowlist, &spec.name)
5206                    && allowlist_permits(thread_allowlist, &spec.name)
5207                    && !external_tools.iter().any(|tool| tool.name == spec.name)
5208            })
5209            .collect::<Vec<_>>();
5210        specs.extend(external_tools.iter().cloned());
5211        specs
5212    }
5213
5214    fn task_ledger_tool_specs(
5215        &self,
5216        profile: Option<&ModelHarnessProfile>,
5217    ) -> Vec<roder_api::tools::ToolSpec> {
5218        self.tool_registry
5219            .get(TASK_LEDGER_TOOL_NAME)
5220            .map(|tool| {
5221                tool.spec()
5222                    .normalized_for_model_profile(schema_policy_for_model(profile))
5223            })
5224            .into_iter()
5225            .collect()
5226    }
5227
5228    pub(crate) fn engine_for(&self, provider: &str) -> anyhow::Result<Arc<dyn InferenceEngine>> {
5229        self.registry
5230            .inference_engine(provider)
5231            .or_else(|| {
5232                self.registry
5233                    .default_inference_engine()
5234                    .filter(|engine| provider.is_empty() || engine.id() == provider)
5235            })
5236            .ok_or_else(|| anyhow::anyhow!("inference provider {provider:?} is not registered"))
5237    }
5238
5239    pub async fn emit(&self, event: RoderEvent) -> EventEnvelope {
5240        if let Some(record) = Self::lifecycle_record_for_event(&event) {
5241            let _ = self.persist_turn_lifecycle_record(&record).await;
5242        }
5243        let envelope = self.bus.emit(event);
5244        if let (Some(store), Some(thread_id)) = (&self.thread_store, envelope.thread_id.as_ref())
5245            && should_persist_thread_event(thread_id)
5246        {
5247            let _ = store.append_event(thread_id, &envelope).await;
5248        }
5249        // Registered event sinks (e.g. process extensions) receive the
5250        // persisted envelope through bounded per-sink queues; a slow sink
5251        // never blocks emit or turn progress.
5252        let dispatcher = self
5253            .event_sink_dispatcher
5254            .get_or_init(|| async {
5255                crate::event_sink_dispatch::EventSinkDispatcher::start(
5256                    &self.registry.event_sinks,
5257                    self.bus.clone(),
5258                )
5259            })
5260            .await;
5261        if !dispatcher.is_empty() {
5262            dispatcher.dispatch(&envelope, &self.bus);
5263        }
5264        envelope
5265    }
5266
5267    /// Records a projected thread item event and persists it through the configured thread store.
5268    ///
5269    /// App-server protocol notification bridges currently call this after translating runtime
5270    /// events into item events. Headless runtime consumers that subscribe to `RoderEvent` directly
5271    /// must make the same call if they need the derived item event stream persisted.
5272    pub async fn record_thread_item_event_kind(
5273        &self,
5274        thread_id: &ThreadId,
5275        turn_id: &TurnId,
5276        timestamp: OffsetDateTime,
5277        kind: ThreadItemEventKind,
5278    ) -> anyhow::Result<ThreadItemEvent> {
5279        let seq = self.next_thread_item_event_seq(thread_id).await?;
5280        let item_event = ThreadItemEvent {
5281            seq,
5282            event_id: format!("{turn_id}-item-event-{seq}"),
5283            thread_id: thread_id.clone(),
5284            turn_id: turn_id.clone(),
5285            timestamp,
5286            event: kind,
5287        };
5288        if let Some(store) = &self.thread_store {
5289            store.append_item_event(thread_id, &item_event).await?;
5290        }
5291        self.remember_thread_item_event(&item_event).await?;
5292        Ok(item_event)
5293    }
5294
5295    async fn next_thread_item_event_seq(&self, thread_id: &ThreadId) -> anyhow::Result<u64> {
5296        self.ensure_thread_item_cache(thread_id).await?;
5297        Ok(self
5298            .thread_item_cache
5299            .lock()
5300            .await
5301            .next_item_event_seq(thread_id))
5302    }
5303
5304    pub async fn thread_item_exists(
5305        &self,
5306        thread_id: &ThreadId,
5307        turn_id: &TurnId,
5308        item_id: &str,
5309    ) -> anyhow::Result<bool> {
5310        self.ensure_thread_item_cache(thread_id).await?;
5311        Ok(self
5312            .thread_item_cache
5313            .lock()
5314            .await
5315            .thread_item_exists(thread_id, turn_id, item_id))
5316    }
5317
5318    pub async fn current_reasoning_item_id(
5319        &self,
5320        thread_id: &ThreadId,
5321        turn_id: &TurnId,
5322    ) -> anyhow::Result<Option<String>> {
5323        self.ensure_thread_item_cache(thread_id).await?;
5324        Ok(self
5325            .thread_item_cache
5326            .lock()
5327            .await
5328            .current_reasoning_item_id(thread_id, turn_id))
5329    }
5330
5331    async fn remember_thread_item_event(&self, item_event: &ThreadItemEvent) -> anyhow::Result<()> {
5332        self.ensure_thread_item_cache(&item_event.thread_id).await?;
5333        self.thread_item_cache
5334            .lock()
5335            .await
5336            .remember_item_event(item_event);
5337        Ok(())
5338    }
5339
5340    pub async fn latest_transcript_item_index(
5341        &self,
5342        thread_id: &ThreadId,
5343        turn_id: &TurnId,
5344    ) -> anyhow::Result<Option<usize>> {
5345        self.ensure_thread_item_cache(thread_id).await?;
5346        Ok(self
5347            .thread_item_cache
5348            .lock()
5349            .await
5350            .latest_transcript_item_index(thread_id, turn_id))
5351    }
5352
5353    async fn next_transcript_item_index(
5354        &self,
5355        thread_id: &ThreadId,
5356        turn_id: &TurnId,
5357    ) -> anyhow::Result<usize> {
5358        self.ensure_thread_item_cache(thread_id).await?;
5359        Ok(self
5360            .thread_item_cache
5361            .lock()
5362            .await
5363            .next_transcript_item_index(thread_id, turn_id))
5364    }
5365
5366    async fn remember_transcript_item_index(
5367        &self,
5368        thread_id: &ThreadId,
5369        turn_id: &TurnId,
5370        item_index: usize,
5371    ) -> anyhow::Result<()> {
5372        self.ensure_thread_item_cache(thread_id).await?;
5373        self.thread_item_cache
5374            .lock()
5375            .await
5376            .remember_transcript_item_index(thread_id, turn_id, item_index);
5377        Ok(())
5378    }
5379
5380    async fn ensure_thread_item_cache(&self, thread_id: &ThreadId) -> anyhow::Result<()> {
5381        if self
5382            .thread_item_cache
5383            .lock()
5384            .await
5385            .contains_thread(thread_id)
5386        {
5387            return Ok(());
5388        }
5389
5390        let snapshot = if let Some(store) = &self.thread_store {
5391            store.load_thread(thread_id).await?
5392        } else {
5393            None
5394        };
5395        self.thread_item_cache.lock().await.ensure_thread(
5396            thread_id,
5397            ThreadItemCacheEntry::from_snapshot(snapshot.as_ref()),
5398        );
5399        Ok(())
5400    }
5401
5402    pub(crate) async fn persist_turn_item(
5403        &self,
5404        thread_id: &ThreadId,
5405        turn_id: &TurnId,
5406        item: &TranscriptItem,
5407    ) -> anyhow::Result<()> {
5408        let item_index = self.next_transcript_item_index(thread_id, turn_id).await?;
5409        let timestamp = OffsetDateTime::now_utc();
5410        self.emit(RoderEvent::TranscriptItemAppended(TranscriptItemAppended {
5411            thread_id: thread_id.clone(),
5412            turn_id: turn_id.clone(),
5413            item_type: match item {
5414                TranscriptItem::UserMessage(_) => "user_message",
5415                TranscriptItem::AssistantMessage(_) => "assistant_message",
5416                TranscriptItem::ReasoningSummary(_) => "reasoning_summary",
5417                TranscriptItem::ToolCall(_) => "tool_call",
5418                TranscriptItem::ToolResult(_) => "tool_result",
5419                TranscriptItem::FileChange(_) => "file_change",
5420                TranscriptItem::ContextCompaction(_) => "context_compaction",
5421                TranscriptItem::Error(_) => "error",
5422                TranscriptItem::ProviderMetadata(_) => "provider_metadata",
5423            }
5424            .to_string(),
5425            item_index: Some(item_index),
5426            item: Some(item.clone()),
5427            timestamp,
5428        }))
5429        .await;
5430        self.remember_transcript_item_index(thread_id, turn_id, item_index)
5431            .await?;
5432        Ok(())
5433    }
5434}
5435
5436fn transcript_has_images(transcript: &[TranscriptItem]) -> bool {
5437    transcript.iter().any(|item| {
5438        matches!(
5439            item,
5440            TranscriptItem::UserMessage(message) if !message.images.is_empty()
5441        )
5442    })
5443}
5444
5445fn transcript_has_task_ledger(transcript: &[TranscriptItem]) -> bool {
5446    transcript.iter().any(|item| {
5447        matches!(
5448            item,
5449            TranscriptItem::ToolResult(result)
5450                if result.name.as_deref() == Some(TASK_LEDGER_TOOL_NAME) && !result.is_error
5451        )
5452    })
5453}
5454
5455fn task_ledger_completion_prompt(transcript: &[TranscriptItem]) -> Option<String> {
5456    let latest = transcript.iter().rev().find_map(|item| match item {
5457        TranscriptItem::ToolResult(result)
5458            if result.name.as_deref() == Some(TASK_LEDGER_TOOL_NAME) && !result.is_error =>
5459        {
5460            Some(result.result.as_str())
5461        }
5462        _ => None,
5463    })?;
5464    if !task_ledger_has_open_items(latest) {
5465        return None;
5466    }
5467
5468    let mut ledger = latest.chars().take(1500).collect::<String>();
5469    if latest.chars().nth(1500).is_some() {
5470        ledger.push_str("...");
5471    }
5472    Some(format!(
5473        "Task Ledger Completion Required: the latest task ledger still has pending or in-progress items. Do not provide a final answer yet. Use tools to complete the remaining scoreable work, create or update any required output files, then call `{TASK_LEDGER_TOOL_NAME}` with every task completed and evidence before finalizing.\n\nLatest ledger:\n{ledger}"
5474    ))
5475}
5476
5477fn task_ledger_deadline_completion_prompt(
5478    remaining_seconds: u64,
5479    reserve_seconds: u64,
5480    completion_prompt: &str,
5481) -> String {
5482    format!(
5483        "Eval deadline scoreable completion: {remaining_seconds} seconds remain in the {reserve_seconds}-second finalization reserve. Do not browse, search, or start slow work. Use the available tools now to create or update the required scoreable output files, run only a quick local check if needed, then update the task ledger to completed before finalizing.\n\n{completion_prompt}"
5484    )
5485}
5486
5487fn task_ledger_scoreable_checkpoint_prompt(
5488    remaining_seconds: u64,
5489    completion_prompt: &str,
5490) -> String {
5491    format!(
5492        "Scoreable Output Checkpoint: {remaining_seconds} seconds remain before the eval deadline. Before any further research, browsing, or long commands, use tools now to ensure the required output file(s) exist with the best evidence-backed answer, even if provisional. If a scoreable file already exists, read it and preserve that candidate unless you have stronger task-specific evidence for a replacement. Do not overwrite a plausible dated, historical, or local-evidence candidate with a current live-page, partial-coverage, or weaker guess merely to refresh the checkpoint. You may continue refining afterward, but do not apologize or finalize until the scoreable file exists and the task ledger is updated.\n\n{completion_prompt}"
5493    )
5494}
5495
5496fn task_ledger_has_open_items(ledger: &str) -> bool {
5497    ledger.lines().any(|line| {
5498        let line = line.trim_start();
5499        line.starts_with("- pending:") || line.starts_with("- in_progress:")
5500    })
5501}
5502
5503fn turn_deadline_for_config(cfg: &RuntimeConfig) -> Option<OffsetDateTime> {
5504    cfg.turn_deadline_seconds
5505        .filter(|seconds| *seconds > 0)
5506        .map(|seconds| OffsetDateTime::now_utc() + Duration::seconds(seconds as i64))
5507}
5508
5509fn deadline_expired(deadline: OffsetDateTime) -> bool {
5510    OffsetDateTime::now_utc() >= deadline
5511}
5512
5513pub(crate) fn deadline_remaining_seconds(deadline: Option<OffsetDateTime>) -> Option<u64> {
5514    let deadline = deadline?;
5515    if deadline <= OffsetDateTime::now_utc() {
5516        return Some(0);
5517    }
5518    Some(
5519        (deadline - OffsetDateTime::now_utc())
5520            .unsigned_abs()
5521            .as_secs()
5522            .max(1),
5523    )
5524}
5525
5526fn deadline_instant(deadline: OffsetDateTime) -> tokio::time::Instant {
5527    let now = OffsetDateTime::now_utc();
5528    if deadline <= now {
5529        return tokio::time::Instant::now();
5530    }
5531    tokio::time::Instant::now() + (deadline - now).unsigned_abs()
5532}
5533
5534fn inference_timeout_deadline(
5535    deadline: Option<OffsetDateTime>,
5536    runtime_profile: RuntimeProfile,
5537    task_ledger_required: bool,
5538    reserve_seconds: u64,
5539    finalization_requested: bool,
5540    task_ledger_scoreable_checkpoints: u8,
5541    transcript: &[TranscriptItem],
5542) -> Option<(OffsetDateTime, InferenceTimeoutAction)> {
5543    let deadline = deadline?;
5544    if runtime_profile == RuntimeProfile::Eval
5545        && task_ledger_required
5546        && !finalization_requested
5547        && task_ledger_scoreable_checkpoints < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
5548        && TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS > reserve_seconds
5549        && task_ledger_completion_prompt(transcript).is_some()
5550    {
5551        let checkpoint_deadline =
5552            deadline - Duration::seconds(TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS as i64);
5553        if checkpoint_deadline > OffsetDateTime::now_utc() {
5554            return Some((
5555                checkpoint_deadline,
5556                InferenceTimeoutAction::ScoreableCheckpoint,
5557            ));
5558        }
5559    }
5560    if runtime_profile == RuntimeProfile::Eval && !finalization_requested {
5561        return Some((
5562            deadline - Duration::seconds(reserve_seconds as i64),
5563            InferenceTimeoutAction::Finalization,
5564        ));
5565    }
5566    Some((deadline, InferenceTimeoutAction::Finalization))
5567}
5568
5569fn turn_partial_result(transcript: &[TranscriptItem]) -> String {
5570    let tool_results = transcript
5571        .iter()
5572        .filter(|item| matches!(item, TranscriptItem::ToolResult(_)))
5573        .count();
5574    let assistant_messages = transcript
5575        .iter()
5576        .filter(|item| matches!(item, TranscriptItem::AssistantMessage(_)))
5577        .count();
5578    format!(
5579        "partial turn state: {} transcript items, {assistant_messages} assistant messages, {tool_results} tool results",
5580        transcript.len()
5581    )
5582}
5583
5584fn reasoning_for_model(cfg: &RuntimeConfig, model: &str) -> ReasoningConfig {
5585    let level = effective_reasoning_for_model(cfg, model);
5586    match level.as_str() {
5587        "" | REASONING_NONE => ReasoningConfig::default(),
5588        level => ReasoningConfig {
5589            enabled: true,
5590            level: Some(level.to_string()),
5591        },
5592    }
5593}
5594
5595fn server_side_compaction_threshold(cfg: &RuntimeConfig, model: &str) -> Option<u32> {
5596    let entry = lookup_model(model)?;
5597    if !entry.supports_compaction {
5598        return None;
5599    }
5600    cfg.auto_compact_token_limit
5601        .or_else(|| {
5602            model_profile_for_model(cfg, model).and_then(|profile| profile.auto_compact_token_limit)
5603        })
5604        .or(Some(entry.auto_compact_token_limit))
5605        .filter(|threshold| *threshold > 0)
5606}
5607
5608pub(crate) fn tool_search_for_provider_model(
5609    cfg: &RuntimeConfig,
5610    provider: &str,
5611    model: &str,
5612) -> ToolSearchConfig {
5613    let mut resolved = cfg.tool_search.clone();
5614    if let Some(provider_config) = cfg.provider_tool_search.get(provider) {
5615        provider_config.apply_to(&mut resolved);
5616    }
5617    if let Some(model_config) = cfg.model_tool_search.get(model) {
5618        model_config.apply_to(&mut resolved);
5619    }
5620    resolved
5621}
5622
5623fn parallel_tool_calls_for_model(cfg: &RuntimeConfig, model: &str) -> bool {
5624    cfg.model_parallel_tool_calls
5625        .get(model)
5626        .copied()
5627        .or_else(|| {
5628            model_profile_for_model(cfg, model).and_then(|profile| profile.parallel_tool_calls)
5629        })
5630        .unwrap_or(true)
5631}
5632
5633/// Count the children an `agent_swarm` call will launch from its arguments
5634/// (item-based spawns plus resumes). Lenient: returns 0 on malformed input.
5635fn agent_swarm_child_count(arguments: &str) -> usize {
5636    serde_json::from_str::<roder_api::subagents::AgentSwarmRequest>(arguments)
5637        .map(|request| request.items.len() + request.resume_agent_ids.len())
5638        .unwrap_or(0)
5639}
5640
5641/// Parse `completed`/`failed`/`aborted` counts from an `<agent_swarm_result>`
5642/// summary line, e.g. `<summary>completed: 2, failed: 1</summary>`. Omitted
5643/// buckets default to 0. Returns `None` when the text is not a swarm result.
5644fn parse_swarm_counts(text: &str) -> Option<(usize, usize, usize)> {
5645    if !text.contains("<agent_swarm_result>") {
5646        return None;
5647    }
5648    let bucket = |label: &str| -> usize {
5649        let needle = format!("{label}: ");
5650        text.find(&needle)
5651            .map(|start| start + needle.len())
5652            .map(|start| {
5653                text[start..]
5654                    .chars()
5655                    .take_while(|c| c.is_ascii_digit())
5656                    .collect::<String>()
5657            })
5658            .and_then(|digits| digits.parse().ok())
5659            .unwrap_or(0)
5660    };
5661    Some((bucket("completed"), bucket("failed"), bucket("aborted")))
5662}
5663
5664fn effective_reasoning_for_model(cfg: &RuntimeConfig, model: &str) -> String {
5665    let base_reasoning = default_effective_reasoning_for_model(cfg, model);
5666    if cfg.dynamic_workflows.effort_profile == DynamicWorkflowEffortProfile::Ultracode {
5667        return ultracode_reasoning_level_for_model(
5668            model,
5669            &cfg.speed_policy.ultracode_reasoning,
5670            &base_reasoning,
5671        );
5672    }
5673    base_reasoning
5674}
5675
5676fn default_effective_reasoning_for_model(cfg: &RuntimeConfig, model: &str) -> String {
5677    let Some(entry) = lookup_model(model) else {
5678        return cfg
5679            .reasoning
5680            .clone()
5681            .unwrap_or_else(|| REASONING_NONE.to_string());
5682    };
5683    if entry.supported_reasoning.is_empty() {
5684        return REASONING_NONE.to_string();
5685    }
5686    cfg.reasoning
5687        .as_deref()
5688        .filter(|reasoning| {
5689            entry
5690                .supported_reasoning
5691                .iter()
5692                .any(|option| option.effort == *reasoning)
5693        })
5694        .map(str::to_string)
5695        .or_else(|| {
5696            model_profile_for_model(cfg, model)
5697                .and_then(|profile| profile.reasoning.orientation)
5698                .filter(|reasoning| {
5699                    entry
5700                        .supported_reasoning
5701                        .iter()
5702                        .any(|option| option.effort == reasoning)
5703                })
5704        })
5705        .unwrap_or_else(|| entry.default_reasoning.to_string())
5706}
5707
5708fn validate_reasoning_effort(model: &str, effort: &str) -> anyhow::Result<()> {
5709    if effort == REASONING_NONE && !model_supports_reasoning(model, effort) {
5710        return Ok(());
5711    }
5712    let Some(entry) = lookup_model(model) else {
5713        return Ok(());
5714    };
5715    if entry
5716        .supported_reasoning
5717        .iter()
5718        .any(|option| option.effort == effort)
5719    {
5720        Ok(())
5721    } else {
5722        anyhow::bail!("model {model} does not support reasoning effort {effort}")
5723    }
5724}
5725
5726fn validate_runtime_config_reasoning(cfg: &RuntimeConfig) -> anyhow::Result<()> {
5727    let Some(reasoning) = cfg.reasoning.as_deref() else {
5728        return Ok(());
5729    };
5730    let Some(entry) = lookup_model(&cfg.default_model) else {
5731        return Ok(());
5732    };
5733    if entry.provider != PROVIDER_GEMINI {
5734        return Ok(());
5735    }
5736    validate_reasoning_effort(&cfg.default_model, reasoning)
5737}
5738
5739fn validate_runtime_inference_router_config(
5740    registry: &ExtensionRegistry,
5741    cfg: &RuntimeConfig,
5742) -> anyhow::Result<()> {
5743    if !cfg.inference_router.enabled {
5744        return Ok(());
5745    }
5746    let Some(router_id) = cfg.inference_router.router_id.as_deref() else {
5747        anyhow::bail!("inference_router.enabled requires inference_router.router");
5748    };
5749    if registry.inference_router(router_id).is_some() {
5750        return Ok(());
5751    }
5752    let available = registry
5753        .inference_routers
5754        .iter()
5755        .map(|router| router.id())
5756        .collect::<Vec<_>>()
5757        .join(", ");
5758    if available.is_empty() {
5759        anyhow::bail!("inference router {router_id:?} is not registered");
5760    }
5761    anyhow::bail!(
5762        "inference router {router_id:?} is not registered; available routers: {available}"
5763    );
5764}
5765
5766fn model_supports_reasoning(model: &str, effort: &str) -> bool {
5767    lookup_model(model)
5768        .map(|entry| {
5769            entry
5770                .supported_reasoning
5771                .iter()
5772                .any(|option| option.effort == effort)
5773        })
5774        .unwrap_or(false)
5775}
5776
5777fn is_final_answer_phase(phase: Option<&str>) -> bool {
5778    phase.is_none_or(|phase| phase.is_empty() || phase == FINAL_ANSWER_PHASE)
5779}
5780
5781fn edit_tool_for_model<'a>(cfg: &'a RuntimeConfig, model: &'a str) -> Option<&'a str> {
5782    cfg.model_edit_tools
5783        .get(model)
5784        .map(String::as_str)
5785        .or_else(|| {
5786            cfg.model_profiles
5787                .get(model)
5788                .and_then(|profile| profile.edit_tool.as_deref())
5789        })
5790        .or_else(|| lookup_model(model).and_then(|entry| entry.edit_tool))
5791        .or(Some(EDIT_TOOL_EDIT))
5792}
5793
5794fn model_profile_for_model(cfg: &RuntimeConfig, model: &str) -> Option<ModelHarnessProfile> {
5795    cfg.model_profiles
5796        .get(model)
5797        .cloned()
5798        .or_else(|| built_in_model_profile(model))
5799}
5800
5801pub(crate) fn allowlist_permits(allowlist: &[String], tool_name: &str) -> bool {
5802    allowlist.is_empty() || allowlist.iter().any(|allowed| allowed == tool_name)
5803}
5804
5805/// Provider-aware profile resolution for the active turn.
5806///
5807/// Many model ids are shared across providers (for example Cursor proxies
5808/// `claude-opus-4-8`). Resolving the harness profile by id alone picks the
5809/// first catalog entry, which assigns cross-provider ids the wrong family and
5810/// instruction overlay (e.g. a `cursor/claude-opus-4-8` turn would otherwise
5811/// inherit the Anthropic overlay). Prefer the explicit `(provider, model)`
5812/// catalog entry, keeping user-configured profile overrides as the top
5813/// precedence.
5814fn model_profile_for_provider_model(
5815    cfg: &RuntimeConfig,
5816    provider: &str,
5817    model: &str,
5818) -> Option<ModelHarnessProfile> {
5819    cfg.model_profiles
5820        .get(model)
5821        .cloned()
5822        .or_else(|| built_in_model_profile_for_provider(provider, model))
5823}
5824
5825fn schema_policy_for_model(profile: Option<&ModelHarnessProfile>) -> ModelSchemaPolicy {
5826    profile
5827        .map(|profile| profile.schema_policy)
5828        .unwrap_or_default()
5829}
5830
5831fn model_profile_segment_metadata(
5832    profile: Option<&ModelHarnessProfile>,
5833    provider: &str,
5834    model: &str,
5835    segment: &str,
5836) -> serde_json::Value {
5837    serde_json::json!({
5838        "kind": MODEL_PROFILE_TRACE_KIND,
5839        "segment": segment,
5840        "provider": provider,
5841        "model": model,
5842        "profileModel": profile.map(|profile| profile.model.as_str()).unwrap_or(model),
5843        "providerFamily": profile.map(|profile| profile.provider_family),
5844        "editTool": profile.and_then(|profile| profile.edit_tool.as_deref()),
5845        "schemaPolicy": profile.map(|profile| profile.schema_policy),
5846        "instructionOverlay": profile.map(|profile| profile.instruction_overlay),
5847        "parallelToolCalls": profile.and_then(|profile| profile.parallel_tool_calls),
5848        "autoCompactTokenLimit": profile.and_then(|profile| profile.auto_compact_token_limit),
5849    })
5850}
5851
5852fn model_switch_summary(
5853    transcript: &[TranscriptItem],
5854    profile: Option<&ModelHarnessProfile>,
5855    provider: &str,
5856    model: &str,
5857    tools: &[roder_api::tools::ToolSpec],
5858) -> Option<String> {
5859    let previous = latest_model_profile_segment(transcript)?;
5860    let previous_model = previous
5861        .get("model")
5862        .and_then(serde_json::Value::as_str)
5863        .unwrap_or_default();
5864    let previous_provider = previous
5865        .get("provider")
5866        .and_then(serde_json::Value::as_str)
5867        .unwrap_or_default();
5868    if previous_model == model && previous_provider == provider {
5869        return None;
5870    }
5871
5872    let previous_profile = previous
5873        .get("profileModel")
5874        .and_then(serde_json::Value::as_str)
5875        .unwrap_or(previous_model);
5876    let current_profile = profile
5877        .map(|profile| profile.model.as_str())
5878        .unwrap_or(model);
5879    let previous_edit_tool = previous
5880        .get("editTool")
5881        .and_then(serde_json::Value::as_str)
5882        .unwrap_or("none");
5883    let current_edit_tool = profile
5884        .and_then(|profile| profile.edit_tool.as_deref())
5885        .unwrap_or("none");
5886    let tool_names = tools
5887        .iter()
5888        .map(|tool| tool.name.as_str())
5889        .take(12)
5890        .collect::<Vec<_>>()
5891        .join(", ");
5892    Some(format!(
5893        "{MODEL_SWITCH_SUMMARY_PREFIX} previous profile {previous_provider}/{previous_profile} used edit tool {previous_edit_tool}. Current profile {provider}/{current_profile} uses edit tool {current_edit_tool}. Available tools now: {}.",
5894        if tool_names.is_empty() {
5895            "none"
5896        } else {
5897            &tool_names
5898        }
5899    ))
5900}
5901
5902fn latest_model_profile_segment(transcript: &[TranscriptItem]) -> Option<&serde_json::Value> {
5903    transcript.iter().rev().find_map(|item| {
5904        let TranscriptItem::ProviderMetadata(value) = item else {
5905            return None;
5906        };
5907        (value.get("kind").and_then(serde_json::Value::as_str) == Some(MODEL_PROFILE_TRACE_KIND))
5908            .then_some(value)
5909    })
5910}
5911
5912pub fn validate_edit_tool(value: &str) -> anyhow::Result<()> {
5913    match value.trim() {
5914        EDIT_TOOL_PATCH | EDIT_TOOL_EDIT => Ok(()),
5915        _ => anyhow::bail!(
5916            "unsupported edit_tool {value:?}; allowed values: {EDIT_TOOL_PATCH}, {EDIT_TOOL_EDIT}"
5917        ),
5918    }
5919}
5920
5921fn should_persist_thread_event(thread_id: &str) -> bool {
5922    !is_synthetic_event_thread_id(thread_id)
5923}
5924
5925#[cfg(test)]
5926#[path = "runtime/codex_v2_tests.rs"]
5927mod codex_v2_tests;
5928
5929#[cfg(test)]
5930#[path = "runtime/codex_v2_lifecycle_tests.rs"]
5931mod codex_v2_lifecycle_tests;
5932
5933#[cfg(test)]
5934mod tests {
5935    use super::*;
5936    use futures::stream;
5937    use roder_api::catalog::{
5938        PROVIDER_MOCK, REASONING_HIGH, REASONING_LOW, REASONING_MEDIUM, REASONING_MINIMAL,
5939        REASONING_NONE, REASONING_XHIGH,
5940    };
5941    use roder_api::extension::ExtensionRegistryBuilder;
5942    use roder_api::inference::{
5943        CompletionMetadata, InferenceCapabilities, InferenceEngine, InferenceEventStream,
5944        InferenceProviderContext, InferenceTurnContext, MessageDelta, ModelDescriptor,
5945        ModelInstructionOverlay, ModelProfileReasoning, ModelSchemaPolicy, ProviderFamily,
5946        ReasoningEffortDescriptor,
5947    };
5948    use roder_api::inference_routing::{
5949        InferenceRouter, InferenceRoutingContext, InferenceRoutingDecision, InferenceRoutingOutcome,
5950    };
5951    use roder_api::thread::ThreadStoreFactory;
5952    use roder_api::tools::{ToolContributor, ToolExecutor, ToolSpec};
5953    use roder_ext_jsonl_thread_store::store::JsonlThreadStoreFactory;
5954    use std::sync::Mutex as StdMutex;
5955
5956    fn test_workspace() -> String {
5957        std::env::current_dir().unwrap().display().to_string()
5958    }
5959
5960    #[test]
5961    fn interactive_explicit_turn_deadline_has_bounded_remaining_time() {
5962        let config = RuntimeConfig {
5963            runtime_profile: RuntimeProfile::Interactive,
5964            turn_deadline_seconds: Some(60),
5965            ..RuntimeConfig::default()
5966        };
5967
5968        let deadline = turn_deadline_for_config(&config).expect("configured turn deadline");
5969        let remaining = deadline_remaining_seconds(Some(deadline));
5970
5971        assert!(
5972            matches!(remaining, Some(1..=60)),
5973            "interactive deadline must be positive and no longer than configured: {remaining:?}"
5974        );
5975    }
5976
5977    #[test]
5978    fn interactive_without_turn_deadline_stays_unbounded() {
5979        let config = RuntimeConfig {
5980            runtime_profile: RuntimeProfile::Interactive,
5981            turn_deadline_seconds: None,
5982            ..RuntimeConfig::default()
5983        };
5984
5985        assert_eq!(turn_deadline_for_config(&config), None);
5986    }
5987
5988    struct MetadataMissingStore;
5989
5990    #[async_trait::async_trait]
5991    impl ThreadStore for MetadataMissingStore {
5992        fn id(&self) -> roder_api::thread::ThreadStoreId {
5993            "metadata-missing-store".to_string()
5994        }
5995
5996        async fn create_thread(&self, metadata: ThreadMetadata) -> anyhow::Result<ThreadMetadata> {
5997            Ok(metadata)
5998        }
5999
6000        async fn list_threads(&self) -> anyhow::Result<Vec<ThreadMetadata>> {
6001            Ok(Vec::new())
6002        }
6003
6004        async fn load_thread(
6005            &self,
6006            _thread_id: &ThreadId,
6007        ) -> anyhow::Result<Option<ThreadSnapshot>> {
6008            Ok(Some(ThreadSnapshot {
6009                metadata: None,
6010                ..ThreadSnapshot::default()
6011            }))
6012        }
6013
6014        async fn append_event(
6015            &self,
6016            _thread_id: &ThreadId,
6017            _envelope: &EventEnvelope,
6018        ) -> anyhow::Result<()> {
6019            Ok(())
6020        }
6021    }
6022
6023    struct MetadataMissingStoreFactory;
6024
6025    impl ThreadStoreFactory for MetadataMissingStoreFactory {
6026        fn id(&self) -> roder_api::thread::ThreadStoreId {
6027            "metadata-missing-store".to_string()
6028        }
6029
6030        fn create(&self) -> Arc<dyn ThreadStore> {
6031            Arc::new(MetadataMissingStore)
6032        }
6033    }
6034
6035    #[test]
6036    fn synthetic_app_server_events_are_not_thread_events() {
6037        for thread_id in ["app-server", "runtime", "thread-workflow"] {
6038            assert!(!should_persist_thread_event(thread_id));
6039        }
6040        assert!(should_persist_thread_event("thread-discovery"));
6041        assert!(should_persist_thread_event("thread-plan"));
6042        assert!(should_persist_thread_event("thread-process"));
6043        assert!(should_persist_thread_event("thread-1"));
6044    }
6045
6046    #[test]
6047    fn server_side_compaction_uses_catalog_ninety_percent_default() {
6048        assert_eq!(
6049            server_side_compaction_threshold(&RuntimeConfig::default(), "gpt-5.5"),
6050            Some(945_000)
6051        );
6052        assert_eq!(
6053            server_side_compaction_threshold(&RuntimeConfig::default(), "gpt-5.3-codex-spark"),
6054            Some(115_200)
6055        );
6056    }
6057
6058    #[test]
6059    fn server_side_compaction_respects_explicit_config_override() {
6060        let cfg = RuntimeConfig {
6061            auto_compact_token_limit: Some(123_456),
6062            ..RuntimeConfig::default()
6063        };
6064
6065        assert_eq!(
6066            server_side_compaction_threshold(&cfg, "gpt-5.5"),
6067            Some(123_456)
6068        );
6069    }
6070
6071    #[tokio::test]
6072    async fn pre_request_compaction_runs_when_server_side_model_is_at_context_window() {
6073        let captured = Arc::new(StdMutex::new(None));
6074        let mut builder = ExtensionRegistryBuilder::new();
6075        builder.inference_engine(Arc::new(CapturingEngine {
6076            request: captured.clone(),
6077        }));
6078        let thread_root = std::env::temp_dir().join(format!(
6079            "roder-pre-request-compaction-{}",
6080            uuid::Uuid::new_v4()
6081        ));
6082        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
6083            base_path: thread_root.clone(),
6084        }));
6085        let runtime = Arc::new(
6086            Runtime::new(
6087                builder.build().unwrap(),
6088                RuntimeConfig {
6089                    default_provider: PROVIDER_MOCK.to_string(),
6090                    default_model: "gpt-5.5".to_string(),
6091                    file_backed_dynamic_context: true,
6092                    ..RuntimeConfig::default()
6093                },
6094            )
6095            .unwrap(),
6096        );
6097        let thread_id = runtime
6098            .create_thread(Some("Pre-request compaction".to_string()))
6099            .await
6100            .unwrap()
6101            .thread_id;
6102        let old_turn = "old-turn".to_string();
6103        runtime
6104            .persist_turn_item(
6105                &thread_id,
6106                &old_turn,
6107                &TranscriptItem::UserMessage(UserMessage::text("old context ".repeat(4_300_000))),
6108            )
6109            .await
6110            .unwrap();
6111
6112        let mut events = runtime.subscribe_events();
6113        runtime
6114            .start_turn(StartTurnRequest {
6115                thread_id: thread_id.clone(),
6116                message: "continue".to_string(),
6117                images: Vec::new(),
6118                provider_override: None,
6119                model_override: None,
6120                reasoning_override: None,
6121                workspace: test_workspace(),
6122                instructions: InstructionBundle::default(),
6123                developer_context: None,
6124                task_ledger_required: false,
6125            })
6126            .await
6127            .unwrap();
6128        loop {
6129            let envelope = tokio::time::timeout(std::time::Duration::from_secs(5), events.recv())
6130                .await
6131                .unwrap()
6132                .unwrap();
6133            if envelope.thread_id.as_deref() == Some(&thread_id)
6134                && matches!(envelope.event, RoderEvent::TurnCompleted(_))
6135            {
6136                break;
6137            }
6138        }
6139
6140        let request = captured.lock().unwrap().clone().unwrap();
6141        assert!(
6142            matches!(
6143                request.transcript.first(),
6144                Some(TranscriptItem::ContextCompaction(_))
6145            ),
6146            "provider request should start with a local emergency compaction item"
6147        );
6148        assert!(
6149            request.transcript.len() < 4,
6150            "provider request should not replay the full oversized prior transcript: {:?}",
6151            request.transcript
6152        );
6153
6154        let _ = std::fs::remove_dir_all(thread_root);
6155    }
6156
6157    #[tokio::test]
6158    async fn continue_after_context_window_failure_compacts_before_provider_request() {
6159        let captured = Arc::new(StdMutex::new(None));
6160        let mut builder = ExtensionRegistryBuilder::new();
6161        builder.inference_engine(Arc::new(CapturingEngine {
6162            request: captured.clone(),
6163        }));
6164        let thread_root = std::env::temp_dir().join(format!(
6165            "roder-context-failure-continue-{}",
6166            uuid::Uuid::new_v4()
6167        ));
6168        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
6169            base_path: thread_root.clone(),
6170        }));
6171        let runtime = Arc::new(
6172            Runtime::new(
6173                builder.build().unwrap(),
6174                RuntimeConfig {
6175                    default_provider: PROVIDER_MOCK.to_string(),
6176                    default_model: "gpt-5.5".to_string(),
6177                    file_backed_dynamic_context: true,
6178                    ..RuntimeConfig::default()
6179                },
6180            )
6181            .unwrap(),
6182        );
6183        let thread_id = runtime
6184            .create_thread(Some("Context failure continue".to_string()))
6185            .await
6186            .unwrap()
6187            .thread_id;
6188        let failed_turn = "failed-turn".to_string();
6189        runtime
6190            .persist_turn_item(
6191                &thread_id,
6192                &failed_turn,
6193                &TranscriptItem::UserMessage(UserMessage::text("old work ".repeat(10_000))),
6194            )
6195            .await
6196            .unwrap();
6197        runtime
6198            .persist_turn_item(
6199                &thread_id,
6200                &failed_turn,
6201                &TranscriptItem::Error(ErrorRecord {
6202                    message: "Your input exceeds the context window of this model. Please adjust your input and try again."
6203                        .to_string(),
6204                }),
6205            )
6206            .await
6207            .unwrap();
6208
6209        let mut events = runtime.subscribe_events();
6210        runtime
6211            .start_turn(StartTurnRequest {
6212                thread_id: thread_id.clone(),
6213                message: "continue".to_string(),
6214                images: Vec::new(),
6215                provider_override: None,
6216                model_override: None,
6217                reasoning_override: None,
6218                workspace: test_workspace(),
6219                instructions: InstructionBundle::default(),
6220                developer_context: None,
6221                task_ledger_required: false,
6222            })
6223            .await
6224            .unwrap();
6225        loop {
6226            let envelope = tokio::time::timeout(std::time::Duration::from_secs(5), events.recv())
6227                .await
6228                .unwrap()
6229                .unwrap();
6230            if envelope.thread_id.as_deref() == Some(&thread_id)
6231                && matches!(envelope.event, RoderEvent::TurnCompleted(_))
6232            {
6233                break;
6234            }
6235        }
6236
6237        let request = captured.lock().unwrap().clone().unwrap();
6238        assert!(
6239            matches!(
6240                request.transcript.first(),
6241                Some(TranscriptItem::ContextCompaction(_))
6242            ),
6243            "provider request after context-window failure should start with local compaction"
6244        );
6245        assert!(
6246            request
6247                .transcript
6248                .iter()
6249                .any(|item| matches!(item, TranscriptItem::UserMessage(message) if message.text == "continue")),
6250            "current continue prompt must be preserved: {:?}",
6251            request.transcript
6252        );
6253        assert!(
6254            !request.transcript.iter().any(
6255                |item| matches!(item, TranscriptItem::Error(error) if error.message.contains("context window"))
6256            ),
6257            "raw prior context-window error should be summarized, not replayed: {:?}",
6258            request.transcript
6259        );
6260
6261        let _ = std::fs::remove_dir_all(thread_root);
6262    }
6263
6264    #[tokio::test]
6265    async fn workspace_for_thread_falls_back_when_metadata_is_missing() {
6266        let workspace = test_workspace();
6267        let mut builder = ExtensionRegistryBuilder::new();
6268        builder.inference_engine(Arc::new(FakeInferenceEngine));
6269        builder.thread_store_factory(Arc::new(MetadataMissingStoreFactory));
6270        let runtime = Runtime::new(
6271            builder.build().unwrap(),
6272            RuntimeConfig {
6273                workspace: Some(workspace.clone()),
6274                ..RuntimeConfig::default()
6275            },
6276        )
6277        .unwrap();
6278
6279        let resolved = runtime
6280            .workspace_for_thread(&ThreadId::from("thread-workflow"))
6281            .await
6282            .unwrap();
6283
6284        assert_eq!(resolved, workspace);
6285    }
6286
6287    #[tokio::test]
6288    async fn automations_can_create_project_thread_with_model_overrides() {
6289        let runtime = Runtime::fake().unwrap();
6290        let workspace = std::env::temp_dir().join("project");
6291        let metadata = runtime
6292            .create_thread_with(CreateThreadRequest {
6293                title: Some("Automation: nightly status".to_string()),
6294                workspace: workspace.display().to_string(),
6295                workspace_id: None,
6296                root_id: None,
6297                provider: Some("mock".to_string()),
6298                model: Some("mock".to_string()),
6299                selection_mode: None,
6300                tool_allowlist: Vec::new(),
6301                developer_instructions: None,
6302                external_tools: Vec::new(),
6303                runner: None,
6304            })
6305            .await
6306            .unwrap();
6307
6308        assert_eq!(
6309            metadata.title.as_deref(),
6310            Some("Automation: nightly status")
6311        );
6312        assert_eq!(metadata.workspace, workspace.display().to_string());
6313        assert_eq!(metadata.provider.as_deref(), Some("mock"));
6314        assert_eq!(metadata.model.as_deref(), Some("mock"));
6315    }
6316
6317    #[tokio::test]
6318    async fn prior_provider_compaction_boundary_is_used_on_next_turn() {
6319        let captured = Arc::new(StdMutex::new(None));
6320        let mut builder = ExtensionRegistryBuilder::new();
6321        builder.inference_engine(Arc::new(CapturingEngine {
6322            request: captured.clone(),
6323        }));
6324        let thread_root = std::env::temp_dir().join(format!(
6325            "roder-provider-compaction-boundary-{}",
6326            uuid::Uuid::new_v4()
6327        ));
6328        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
6329            base_path: thread_root.clone(),
6330        }));
6331        let runtime = Arc::new(
6332            Runtime::new(
6333                builder.build().unwrap(),
6334                RuntimeConfig {
6335                    default_provider: PROVIDER_MOCK.to_string(),
6336                    default_model: "gpt-5.5".to_string(),
6337                    ..RuntimeConfig::default()
6338                },
6339            )
6340            .unwrap(),
6341        );
6342        let thread_id = runtime
6343            .create_thread(Some("Provider compaction boundary".to_string()))
6344            .await
6345            .unwrap()
6346            .thread_id;
6347        let old_turn = "old-turn".to_string();
6348        runtime
6349            .persist_turn_item(
6350                &thread_id,
6351                &old_turn,
6352                &TranscriptItem::UserMessage(UserMessage::text(
6353                    "old history that must not be replayed after provider compaction",
6354                )),
6355            )
6356            .await
6357            .unwrap();
6358        runtime
6359            .persist_turn_item(
6360                &thread_id,
6361                &old_turn,
6362                &TranscriptItem::AssistantMessage(AssistantMessage {
6363                    text: "old answer".to_string(),
6364                    phase: None,
6365                }),
6366            )
6367            .await
6368            .unwrap();
6369        runtime
6370            .persist_turn_item(
6371                &thread_id,
6372                &old_turn,
6373                &TranscriptItem::ProviderMetadata(serde_json::json!({
6374                    "output": [{
6375                        "id": "cmp_1",
6376                        "type": "compaction",
6377                        "encrypted_content": "opaque-state"
6378                    }]
6379                })),
6380            )
6381            .await
6382            .unwrap();
6383        runtime
6384            .persist_turn_item(
6385                &thread_id,
6386                &old_turn,
6387                &TranscriptItem::AssistantMessage(AssistantMessage {
6388                    text: "after compact".to_string(),
6389                    phase: None,
6390                }),
6391            )
6392            .await
6393            .unwrap();
6394
6395        let mut events = runtime.subscribe_events();
6396        runtime
6397            .start_turn(StartTurnRequest {
6398                thread_id: thread_id.clone(),
6399                message: "continue".to_string(),
6400                images: Vec::new(),
6401                provider_override: None,
6402                model_override: None,
6403                reasoning_override: None,
6404                workspace: test_workspace(),
6405                instructions: InstructionBundle::default(),
6406                developer_context: None,
6407                task_ledger_required: false,
6408            })
6409            .await
6410            .unwrap();
6411        loop {
6412            let envelope = tokio::time::timeout(std::time::Duration::from_secs(5), events.recv())
6413                .await
6414                .unwrap()
6415                .unwrap();
6416            if envelope.thread_id.as_deref() == Some(&thread_id)
6417                && matches!(envelope.event, RoderEvent::TurnCompleted(_))
6418            {
6419                break;
6420            }
6421        }
6422
6423        let request = captured.lock().unwrap().clone().unwrap();
6424        let compaction_idx = request.transcript.iter().position(|item| {
6425            matches!(
6426                item,
6427                TranscriptItem::ProviderMetadata(metadata)
6428                    if crate::compaction::provider_metadata_has_compaction(metadata)
6429            )
6430        });
6431        assert!(
6432            compaction_idx.is_some(),
6433            "next provider request should include the provider compaction item: {:?}",
6434            request.transcript
6435        );
6436        // Skill/context injectors may prepend non-history items, but no prior-turn
6437        // conversation may appear before the latest provider compaction boundary.
6438        let history_before_boundary =
6439            request
6440                .transcript
6441                .iter()
6442                .take(compaction_idx.unwrap())
6443                .any(|item| match item {
6444                    TranscriptItem::AssistantMessage(_)
6445                    | TranscriptItem::ToolCall(_)
6446                    | TranscriptItem::ToolResult(_) => true,
6447                    TranscriptItem::UserMessage(message) => {
6448                        !message.text.contains("<skills>") && message.text != "continue"
6449                    }
6450                    _ => false,
6451                });
6452        assert!(
6453            !history_before_boundary,
6454            "pre-provider-compaction conversation must not be replayed: {:?}",
6455            request.transcript
6456        );
6457        assert!(
6458            !request.transcript.iter().any(|item| {
6459                matches!(
6460                    item,
6461                    TranscriptItem::UserMessage(message)
6462                        if message.text.contains("old history that must not be replayed")
6463                )
6464            }),
6465            "pre-provider-compaction history must not be replayed: {:?}",
6466            request.transcript
6467        );
6468        assert!(
6469            request.transcript.iter().any(|item| {
6470                matches!(
6471                    item,
6472                    TranscriptItem::UserMessage(message) if message.text == "continue"
6473                )
6474            }),
6475            "current continue prompt must be preserved: {:?}",
6476            request.transcript
6477        );
6478        assert_eq!(
6479            request.runtime.auto_compact_token_limit,
6480            Some(945_000),
6481            "gpt-5.5 should keep server-side auto compaction configured"
6482        );
6483
6484        let _ = std::fs::remove_dir_all(thread_root);
6485    }
6486
6487    #[test]
6488    fn server_side_compaction_is_only_enabled_for_supported_models() {
6489        let cfg = RuntimeConfig {
6490            auto_compact_token_limit: Some(123_456),
6491            ..RuntimeConfig::default()
6492        };
6493
6494        assert_eq!(server_side_compaction_threshold(&cfg, "mock"), None);
6495        assert_eq!(
6496            server_side_compaction_threshold(&cfg, "codex-auto-review"),
6497            None
6498        );
6499    }
6500
6501    #[test]
6502    fn reasoning_is_disabled_for_models_without_reasoning_support() {
6503        let cfg = RuntimeConfig {
6504            reasoning: Some(REASONING_HIGH.to_string()),
6505            ..RuntimeConfig::default()
6506        };
6507
6508        assert_eq!(
6509            effective_reasoning_for_model(&cfg, "claude-haiku-4-5-20251001"),
6510            REASONING_NONE
6511        );
6512        assert_eq!(
6513            reasoning_for_model(&cfg, "claude-haiku-4-5-20251001"),
6514            ReasoningConfig::default()
6515        );
6516    }
6517
6518    #[test]
6519    fn unsupported_configured_reasoning_falls_back_to_model_default() {
6520        let cfg = RuntimeConfig {
6521            reasoning: Some(REASONING_MINIMAL.to_string()),
6522            ..RuntimeConfig::default()
6523        };
6524
6525        assert_eq!(
6526            effective_reasoning_for_model(&cfg, "gpt-5.5"),
6527            REASONING_MEDIUM
6528        );
6529    }
6530
6531    #[test]
6532    fn unsupported_configured_gemini_reasoning_is_rejected() {
6533        let mut builder = ExtensionRegistryBuilder::new();
6534        builder.inference_engine(std::sync::Arc::new(FakeInferenceEngine));
6535
6536        let err = match Runtime::new(
6537            builder.build().unwrap(),
6538            RuntimeConfig {
6539                default_model: "gemini-3.5-flash".to_string(),
6540                reasoning: Some(REASONING_XHIGH.to_string()),
6541                ..RuntimeConfig::default()
6542            },
6543        ) {
6544            Ok(_) => panic!("expected unsupported Gemini reasoning to be rejected"),
6545            Err(err) => err,
6546        };
6547
6548        assert!(
6549            err.to_string()
6550                .contains("model gemini-3.5-flash does not support reasoning effort xhigh")
6551        );
6552    }
6553
6554    #[tokio::test]
6555    async fn selecting_none_for_non_reasoning_model_preserves_stored_preference() {
6556        let runtime = Runtime::new(
6557            Runtime::fake().unwrap().registry,
6558            RuntimeConfig {
6559                reasoning: Some(REASONING_HIGH.to_string()),
6560                ..RuntimeConfig::default()
6561            },
6562        )
6563        .unwrap();
6564
6565        let cfg = runtime
6566            .select_provider(
6567                roder_api::catalog::PROVIDER_MOCK.to_string(),
6568                Some("claude-haiku-4-5-20251001".to_string()),
6569                Some(REASONING_NONE.to_string()),
6570            )
6571            .await
6572            .unwrap();
6573
6574        assert_eq!(cfg.reasoning.as_deref(), Some(REASONING_HIGH));
6575        assert_eq!(runtime.effective_reasoning().await, REASONING_NONE);
6576    }
6577
6578    #[tokio::test]
6579    async fn selecting_none_for_model_that_supports_none_updates_preference() {
6580        let runtime = Runtime::new(
6581            Runtime::fake().unwrap().registry,
6582            RuntimeConfig {
6583                reasoning: Some(REASONING_HIGH.to_string()),
6584                ..RuntimeConfig::default()
6585            },
6586        )
6587        .unwrap();
6588
6589        let cfg = runtime
6590            .select_provider(
6591                roder_api::catalog::PROVIDER_MOCK.to_string(),
6592                Some("mock".to_string()),
6593                Some(REASONING_NONE.to_string()),
6594            )
6595            .await
6596            .unwrap();
6597
6598        assert_eq!(cfg.reasoning.as_deref(), Some(REASONING_NONE));
6599    }
6600
6601    #[test]
6602    fn parallel_tool_calls_default_on_with_model_override() {
6603        assert!(parallel_tool_calls_for_model(
6604            &RuntimeConfig::default(),
6605            "custom-model"
6606        ));
6607
6608        let cfg = RuntimeConfig {
6609            model_parallel_tool_calls: std::collections::HashMap::from([(
6610                "custom-model".to_string(),
6611                false,
6612            )]),
6613            ..RuntimeConfig::default()
6614        };
6615
6616        assert!(!parallel_tool_calls_for_model(&cfg, "custom-model"));
6617        assert!(parallel_tool_calls_for_model(&cfg, "other-model"));
6618    }
6619
6620    #[test]
6621    fn profile_parallel_tool_calls_applies_between_config_and_default() {
6622        let cfg = RuntimeConfig {
6623            model_profiles: std::collections::HashMap::from([(
6624                "gpt-5.5".to_string(),
6625                test_model_profile("gpt-5.5"),
6626            )]),
6627            ..RuntimeConfig::default()
6628        };
6629
6630        assert!(!parallel_tool_calls_for_model(&cfg, "gpt-5.5"));
6631
6632        let cfg = RuntimeConfig {
6633            model_parallel_tool_calls: std::collections::HashMap::from([(
6634                "gpt-5.5".to_string(),
6635                true,
6636            )]),
6637            ..cfg
6638        };
6639
6640        assert!(parallel_tool_calls_for_model(&cfg, "gpt-5.5"));
6641    }
6642
6643    struct CapturingEngine {
6644        request: Arc<StdMutex<Option<AgentInferenceRequest>>>,
6645    }
6646
6647    #[async_trait::async_trait]
6648    impl InferenceEngine for CapturingEngine {
6649        fn id(&self) -> String {
6650            roder_api::catalog::PROVIDER_MOCK.to_string()
6651        }
6652
6653        fn capabilities(&self) -> InferenceCapabilities {
6654            InferenceCapabilities::coding_agent_default()
6655        }
6656
6657        async fn list_models(
6658            &self,
6659            _ctx: InferenceProviderContext<'_>,
6660        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
6661            Ok(roder_api::catalog::models_for_provider(
6662                roder_api::catalog::PROVIDER_MOCK,
6663                true,
6664            ))
6665        }
6666
6667        async fn stream_turn(
6668            &self,
6669            _ctx: InferenceTurnContext<'_>,
6670            request: AgentInferenceRequest,
6671        ) -> anyhow::Result<InferenceEventStream> {
6672            *self.request.lock().unwrap() = Some(request);
6673            Ok(Box::pin(stream::iter(vec![
6674                Ok(InferenceEvent::MessageDelta(MessageDelta {
6675                    text: "done".to_string(),
6676                    phase: None,
6677                })),
6678                Ok(InferenceEvent::Completed(CompletionMetadata {
6679                    stop_reason: Some("stop".to_string()),
6680                    provider_response_id: None,
6681                })),
6682            ])))
6683        }
6684    }
6685
6686    struct RoutingCaptureEngine {
6687        id: &'static str,
6688        models: Vec<ModelDescriptor>,
6689        requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
6690    }
6691
6692    #[async_trait::async_trait]
6693    impl InferenceEngine for RoutingCaptureEngine {
6694        fn id(&self) -> String {
6695            self.id.to_string()
6696        }
6697
6698        fn capabilities(&self) -> InferenceCapabilities {
6699            InferenceCapabilities::coding_agent_default()
6700        }
6701
6702        async fn list_models(
6703            &self,
6704            _ctx: InferenceProviderContext<'_>,
6705        ) -> anyhow::Result<Vec<ModelDescriptor>> {
6706            Ok(self.models.clone())
6707        }
6708
6709        async fn stream_turn(
6710            &self,
6711            _ctx: InferenceTurnContext<'_>,
6712            request: AgentInferenceRequest,
6713        ) -> anyhow::Result<InferenceEventStream> {
6714            self.requests.lock().unwrap().push(request);
6715            Ok(Box::pin(stream::iter(vec![
6716                Ok(InferenceEvent::MessageDelta(MessageDelta {
6717                    text: "routed".to_string(),
6718                    phase: None,
6719                })),
6720                Ok(InferenceEvent::Completed(CompletionMetadata {
6721                    stop_reason: Some("stop".to_string()),
6722                    provider_response_id: None,
6723                })),
6724            ])))
6725        }
6726    }
6727
6728    struct StaticRouter {
6729        id: &'static str,
6730        decision: InferenceRoutingDecision,
6731        contexts: Arc<StdMutex<Vec<InferenceRoutingContext>>>,
6732    }
6733
6734    #[async_trait::async_trait]
6735    impl InferenceRouter for StaticRouter {
6736        fn id(&self) -> String {
6737            self.id.to_string()
6738        }
6739
6740        async fn route(
6741            &self,
6742            context: InferenceRoutingContext,
6743        ) -> anyhow::Result<InferenceRoutingDecision> {
6744            self.contexts.lock().unwrap().push(context);
6745            Ok(self.decision.clone())
6746        }
6747    }
6748
6749    fn routing_test_model(id: &str, supported_reasoning: &[&str]) -> ModelDescriptor {
6750        ModelDescriptor {
6751            id: id.to_string(),
6752            name: id.to_string(),
6753            context_window: Some(128_000),
6754            default_reasoning: supported_reasoning
6755                .first()
6756                .map(|effort| (*effort).to_string()),
6757            supported_reasoning: supported_reasoning
6758                .iter()
6759                .map(|effort| ReasoningEffortDescriptor {
6760                    effort: (*effort).to_string(),
6761                    description: format!("{effort} reasoning"),
6762                })
6763                .collect(),
6764        }
6765    }
6766
6767    struct TaskLedgerCompletionGateEngine {
6768        calls: StdMutex<u32>,
6769        requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
6770    }
6771
6772    #[async_trait::async_trait]
6773    impl InferenceEngine for TaskLedgerCompletionGateEngine {
6774        fn id(&self) -> String {
6775            roder_api::catalog::PROVIDER_MOCK.to_string()
6776        }
6777
6778        fn capabilities(&self) -> InferenceCapabilities {
6779            InferenceCapabilities::coding_agent_default()
6780        }
6781
6782        async fn list_models(
6783            &self,
6784            _ctx: InferenceProviderContext<'_>,
6785        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
6786            Ok(roder_api::catalog::models_for_provider(
6787                roder_api::catalog::PROVIDER_MOCK,
6788                true,
6789            ))
6790        }
6791
6792        async fn stream_turn(
6793            &self,
6794            _ctx: InferenceTurnContext<'_>,
6795            request: AgentInferenceRequest,
6796        ) -> anyhow::Result<InferenceEventStream> {
6797            self.requests.lock().unwrap().push(request);
6798            let mut calls = self.calls.lock().unwrap();
6799            *calls += 1;
6800            let events = match *calls {
6801                1 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
6802                    id: "ledger-open".to_string(),
6803                    name: TASK_LEDGER_TOOL_NAME.to_string(),
6804                    arguments: serde_json::json!({
6805                        "tasks": [
6806                            {
6807                                "id": "inspect",
6808                                "content": "Inspect local assets",
6809                                "status": "completed",
6810                                "evidence": "listed workspace"
6811                            },
6812                            {
6813                                "id": "write",
6814                                "content": "Write /app/result.txt",
6815                                "status": "pending"
6816                            }
6817                        ],
6818                        "requireCompletionEvidence": true
6819                    })
6820                    .to_string(),
6821                }))],
6822                3 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
6823                    id: "ledger-complete".to_string(),
6824                    name: TASK_LEDGER_TOOL_NAME.to_string(),
6825                    arguments: serde_json::json!({
6826                        "tasks": [
6827                            {
6828                                "id": "inspect",
6829                                "content": "Inspect local assets",
6830                                "status": "completed",
6831                                "evidence": "listed workspace"
6832                            },
6833                            {
6834                                "id": "write",
6835                                "content": "Write /app/result.txt",
6836                                "status": "completed",
6837                                "evidence": "wrote answer"
6838                            }
6839                        ],
6840                        "requireCompletionEvidence": true
6841                    })
6842                    .to_string(),
6843                }))],
6844                _ => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
6845                    text: "final".to_string(),
6846                    phase: None,
6847                }))],
6848            };
6849            Ok(Box::pin(stream::iter(events.into_iter().chain(
6850                std::iter::once(Ok(InferenceEvent::Completed(CompletionMetadata {
6851                    stop_reason: Some("stop".to_string()),
6852                    provider_response_id: None,
6853                }))),
6854            ))))
6855        }
6856    }
6857
6858    struct VerificationGateEngine {
6859        calls: StdMutex<u32>,
6860    }
6861
6862    #[async_trait::async_trait]
6863    impl InferenceEngine for VerificationGateEngine {
6864        fn id(&self) -> String {
6865            roder_api::catalog::PROVIDER_MOCK.to_string()
6866        }
6867
6868        fn capabilities(&self) -> InferenceCapabilities {
6869            InferenceCapabilities::coding_agent_default()
6870        }
6871
6872        async fn list_models(
6873            &self,
6874            _ctx: InferenceProviderContext<'_>,
6875        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
6876            Ok(roder_api::catalog::models_for_provider(
6877                roder_api::catalog::PROVIDER_MOCK,
6878                true,
6879            ))
6880        }
6881
6882        async fn stream_turn(
6883            &self,
6884            _ctx: InferenceTurnContext<'_>,
6885            request: AgentInferenceRequest,
6886        ) -> anyhow::Result<InferenceEventStream> {
6887            let mut calls = self.calls.lock().unwrap();
6888            *calls += 1;
6889            let events = match *calls {
6890                1 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
6891                    id: "write-1".to_string(),
6892                    name: "write_file".to_string(),
6893                    arguments: serde_json::json!({
6894                        "path": "src/lib.rs",
6895                        "content": "pub fn answer() -> u8 { 42 }\n"
6896                    })
6897                    .to_string(),
6898                }))],
6899                2 => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
6900                    text: "done too early".to_string(),
6901                    phase: None,
6902                }))],
6903                3 if request.transcript.iter().any(|item| {
6904                    matches!(
6905                        item,
6906                        TranscriptItem::UserMessage(message)
6907                            if message.text.contains("Verification gate blocked final completion")
6908                    )
6909                }) =>
6910                {
6911                    vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
6912                        id: "verify-1".to_string(),
6913                        name: crate::verification_gate::VERIFICATION_TOOL_NAME.to_string(),
6914                        arguments: serde_json::json!({
6915                            "originalTask": "write code",
6916                            "changedFiles": ["src/lib.rs"],
6917                            "toolEvidence": ["write_file wrote src/lib.rs"],
6918                            "testsRun": ["cargo test -p roder-core verification_gate"],
6919                            "openGaps": [],
6920                            "status": "completed"
6921                        })
6922                        .to_string(),
6923                    }))]
6924                }
6925                _ => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
6926                    text: "verified final".to_string(),
6927                    phase: None,
6928                }))],
6929            };
6930            Ok(Box::pin(stream::iter(events.into_iter().chain(
6931                std::iter::once(Ok(InferenceEvent::Completed(CompletionMetadata {
6932                    stop_reason: Some("stop".to_string()),
6933                    provider_response_id: None,
6934                }))),
6935            ))))
6936        }
6937    }
6938
6939    struct SpeedPolicyEngine {
6940        calls: StdMutex<u32>,
6941        requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
6942    }
6943
6944    #[async_trait::async_trait]
6945    impl InferenceEngine for SpeedPolicyEngine {
6946        fn id(&self) -> String {
6947            roder_api::catalog::PROVIDER_MOCK.to_string()
6948        }
6949
6950        fn capabilities(&self) -> InferenceCapabilities {
6951            InferenceCapabilities::coding_agent_default()
6952        }
6953
6954        async fn list_models(
6955            &self,
6956            _ctx: InferenceProviderContext<'_>,
6957        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
6958            Ok(roder_api::catalog::models_for_provider(
6959                roder_api::catalog::PROVIDER_MOCK,
6960                true,
6961            ))
6962        }
6963
6964        async fn stream_turn(
6965            &self,
6966            _ctx: InferenceTurnContext<'_>,
6967            request: AgentInferenceRequest,
6968        ) -> anyhow::Result<InferenceEventStream> {
6969            self.requests.lock().unwrap().push(request.clone());
6970            let mut calls = self.calls.lock().unwrap();
6971            *calls += 1;
6972            let events = match *calls {
6973                1 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
6974                    id: "write-1".to_string(),
6975                    name: "write_file".to_string(),
6976                    arguments: serde_json::json!({
6977                        "path": "src/lib.rs",
6978                        "content": "pub fn answer() -> u8 { 42 }\n"
6979                    })
6980                    .to_string(),
6981                }))],
6982                3 if request.transcript.iter().any(|item| {
6983                    matches!(
6984                        item,
6985                        TranscriptItem::UserMessage(message)
6986                            if message.text.contains("Verification gate blocked final completion")
6987                    )
6988                }) =>
6989                {
6990                    vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
6991                        id: "verify-1".to_string(),
6992                        name: crate::verification_gate::VERIFICATION_TOOL_NAME.to_string(),
6993                        arguments: serde_json::json!({
6994                            "originalTask": "write code",
6995                            "changedFiles": ["src/lib.rs"],
6996                            "toolEvidence": ["write_file wrote src/lib.rs"],
6997                            "testsRun": ["cargo test -p roder-core speed_policy"],
6998                            "openGaps": [],
6999                            "status": "completed"
7000                        })
7001                        .to_string(),
7002                    }))]
7003                }
7004                _ => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
7005                    text: "done".to_string(),
7006                    phase: None,
7007                }))],
7008            };
7009            Ok(Box::pin(stream::iter(events.into_iter().chain(
7010                std::iter::once(Ok(InferenceEvent::Completed(CompletionMetadata {
7011                    stop_reason: Some("stop".to_string()),
7012                    provider_response_id: None,
7013                }))),
7014            ))))
7015        }
7016    }
7017
7018    struct SwitchCaptureEngine {
7019        requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
7020    }
7021
7022    #[async_trait::async_trait]
7023    impl InferenceEngine for SwitchCaptureEngine {
7024        fn id(&self) -> String {
7025            roder_api::catalog::PROVIDER_MOCK.to_string()
7026        }
7027
7028        fn capabilities(&self) -> InferenceCapabilities {
7029            InferenceCapabilities::coding_agent_default()
7030        }
7031
7032        async fn list_models(
7033            &self,
7034            _ctx: InferenceProviderContext<'_>,
7035        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7036            Ok(roder_api::catalog::models_for_provider(
7037                roder_api::catalog::PROVIDER_MOCK,
7038                true,
7039            ))
7040        }
7041
7042        async fn stream_turn(
7043            &self,
7044            _ctx: InferenceTurnContext<'_>,
7045            request: AgentInferenceRequest,
7046        ) -> anyhow::Result<InferenceEventStream> {
7047            self.requests.lock().unwrap().push(request);
7048            Ok(Box::pin(stream::iter(vec![
7049                Ok(InferenceEvent::MessageDelta(MessageDelta {
7050                    text: "done".to_string(),
7051                    phase: None,
7052                })),
7053                Ok(InferenceEvent::Completed(CompletionMetadata {
7054                    stop_reason: Some("stop".to_string()),
7055                    provider_response_id: None,
7056                })),
7057            ])))
7058        }
7059    }
7060
7061    struct DeadlineEngine;
7062
7063    #[async_trait::async_trait]
7064    impl InferenceEngine for DeadlineEngine {
7065        fn id(&self) -> String {
7066            roder_api::catalog::PROVIDER_MOCK.to_string()
7067        }
7068
7069        fn capabilities(&self) -> InferenceCapabilities {
7070            InferenceCapabilities::coding_agent_default()
7071        }
7072
7073        async fn list_models(
7074            &self,
7075            _ctx: InferenceProviderContext<'_>,
7076        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7077            Ok(Vec::new())
7078        }
7079
7080        async fn stream_turn(
7081            &self,
7082            _ctx: InferenceTurnContext<'_>,
7083            _request: AgentInferenceRequest,
7084        ) -> anyhow::Result<InferenceEventStream> {
7085            Ok(Box::pin(stream::once(async {
7086                tokio::time::sleep(std::time::Duration::from_secs(60)).await;
7087                Ok(InferenceEvent::MessageDelta(MessageDelta {
7088                    text: "too late".to_string(),
7089                    phase: None,
7090                }))
7091            })))
7092        }
7093    }
7094
7095    struct WriteFileContributor;
7096
7097    impl ToolContributor for WriteFileContributor {
7098        fn id(&self) -> String {
7099            "test-write".to_string()
7100        }
7101
7102        fn contribute(&self, registry: &mut ToolRegistry) -> anyhow::Result<()> {
7103            registry.register(Arc::new(WriteFileTool))
7104        }
7105    }
7106
7107    struct WriteFileTool;
7108
7109    #[async_trait::async_trait]
7110    impl ToolExecutor for WriteFileTool {
7111        fn spec(&self) -> ToolSpec {
7112            ToolSpec {
7113                name: "write_file".to_string(),
7114                description: "Write a test file.".to_string(),
7115                parameters: serde_json::json!({
7116                    "type": "object",
7117                    "properties": {
7118                        "path": { "type": "string" },
7119                        "content": { "type": "string" }
7120                    },
7121                    "required": ["path", "content"],
7122                    "additionalProperties": false
7123                }),
7124            }
7125        }
7126
7127        async fn execute(
7128            &self,
7129            _ctx: ToolExecutionContext,
7130            call: ToolCall,
7131        ) -> anyhow::Result<ToolResult> {
7132            let path = call
7133                .arguments
7134                .get("path")
7135                .and_then(serde_json::Value::as_str)
7136                .unwrap_or("src/lib.rs");
7137            Ok(ToolResult {
7138                id: call.id,
7139                name: call.name,
7140                text: format!("wrote {path}"),
7141                data: serde_json::json!({ "path": path }),
7142                is_error: false,
7143            })
7144        }
7145    }
7146
7147    struct ProfileToolContributor;
7148
7149    impl ToolContributor for ProfileToolContributor {
7150        fn id(&self) -> String {
7151            "profile-tools".to_string()
7152        }
7153
7154        fn contribute(&self, registry: &mut ToolRegistry) -> anyhow::Result<()> {
7155            for name in ["apply_patch", "edit", "multi_edit", "write_file"] {
7156                registry.register(Arc::new(ProfileTool {
7157                    name: name.to_string(),
7158                }))?;
7159            }
7160            Ok(())
7161        }
7162    }
7163
7164    struct ProfileTool {
7165        name: String,
7166    }
7167
7168    #[async_trait::async_trait]
7169    impl ToolExecutor for ProfileTool {
7170        fn spec(&self) -> ToolSpec {
7171            ToolSpec {
7172                name: self.name.clone(),
7173                description: format!("{} test tool", self.name),
7174                parameters: serde_json::json!({
7175                    "type": "object",
7176                    "properties": {
7177                        "path": { "type": "string" },
7178                        "content": { "type": "string" }
7179                    },
7180                    "required": ["path", "content"],
7181                    "additionalProperties": false
7182                }),
7183            }
7184        }
7185
7186        async fn execute(
7187            &self,
7188            _ctx: ToolExecutionContext,
7189            call: ToolCall,
7190        ) -> anyhow::Result<ToolResult> {
7191            Ok(ToolResult {
7192                id: call.id,
7193                name: call.name,
7194                text: "ok".to_string(),
7195                data: serde_json::json!({}),
7196                is_error: false,
7197            })
7198        }
7199    }
7200
7201    fn test_model_profile(model: &str) -> ModelHarnessProfile {
7202        ModelHarnessProfile {
7203            model: model.to_string(),
7204            provider: roder_api::catalog::PROVIDER_OPENAI.to_string(),
7205            provider_family: ProviderFamily::OpenAi,
7206            edit_tool: Some(EDIT_TOOL_EDIT.to_string()),
7207            schema_policy: ModelSchemaPolicy::StandardRequiredFirst,
7208            instruction_overlay: ModelInstructionOverlay::IntuitiveContext,
7209            reasoning: ModelProfileReasoning {
7210                orientation: Some(REASONING_LOW.to_string()),
7211                execution: Some(REASONING_LOW.to_string()),
7212                verification: Some(REASONING_LOW.to_string()),
7213                recovery: Some(REASONING_LOW.to_string()),
7214            },
7215            parallel_tool_calls: Some(false),
7216            auto_compact_token_limit: Some(123_000),
7217        }
7218    }
7219
7220    async fn captured_profile_request(cfg: RuntimeConfig) -> AgentInferenceRequest {
7221        let captured = Arc::new(StdMutex::new(None));
7222        let mut builder = ExtensionRegistryBuilder::new();
7223        builder.inference_engine(Arc::new(CapturingEngine {
7224            request: captured.clone(),
7225        }));
7226        builder.tool_contributor(Arc::new(ProfileToolContributor));
7227        let runtime = Arc::new(Runtime::new(builder.build().unwrap(), cfg).unwrap());
7228        let mut rx = runtime.subscribe_events();
7229        let turn_id = runtime
7230            .start_turn(StartTurnRequest {
7231                thread_id: "thread-model-profile".to_string(),
7232                message: "use profile knobs".to_string(),
7233                images: Vec::new(),
7234                provider_override: None,
7235                model_override: None,
7236                reasoning_override: None,
7237                workspace: test_workspace(),
7238                instructions: InstructionBundle {
7239                    system: None,
7240                    developer: Some("base developer".to_string()),
7241                    developer_context: None,
7242                },
7243                developer_context: None,
7244                task_ledger_required: false,
7245            })
7246            .await
7247            .unwrap();
7248
7249        tokio::time::timeout(std::time::Duration::from_secs(5), async {
7250            loop {
7251                let envelope = rx.recv().await.unwrap();
7252                if envelope.turn_id.as_deref() != Some(&turn_id) {
7253                    continue;
7254                }
7255                match envelope.event {
7256                    RoderEvent::TurnCompleted(_) => break,
7257                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7258                    _ => {}
7259                }
7260            }
7261        })
7262        .await
7263        .unwrap();
7264
7265        captured.lock().unwrap().clone().unwrap()
7266    }
7267
7268    struct ToolThenStopEngine {
7269        calls: StdMutex<u32>,
7270    }
7271
7272    #[async_trait::async_trait]
7273    impl InferenceEngine for ToolThenStopEngine {
7274        fn id(&self) -> String {
7275            roder_api::catalog::PROVIDER_MOCK.to_string()
7276        }
7277
7278        fn capabilities(&self) -> InferenceCapabilities {
7279            InferenceCapabilities::coding_agent_default()
7280        }
7281
7282        async fn list_models(
7283            &self,
7284            _ctx: InferenceProviderContext<'_>,
7285        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7286            Ok(roder_api::catalog::models_for_provider(
7287                roder_api::catalog::PROVIDER_MOCK,
7288                true,
7289            ))
7290        }
7291
7292        async fn stream_turn(
7293            &self,
7294            _ctx: InferenceTurnContext<'_>,
7295            _request: AgentInferenceRequest,
7296        ) -> anyhow::Result<InferenceEventStream> {
7297            let mut calls = self.calls.lock().unwrap();
7298            *calls += 1;
7299            let events = match *calls {
7300                1 => vec![
7301                    Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7302                        id: "write-1".to_string(),
7303                        name: "write_file".to_string(),
7304                        arguments: serde_json::json!({
7305                            "path": "src/lib.rs",
7306                            "content": "pub fn answer() -> u8 { 42 }\n"
7307                        })
7308                        .to_string(),
7309                    })),
7310                    Ok(InferenceEvent::Completed(CompletionMetadata {
7311                        stop_reason: Some("tool_use".to_string()),
7312                        provider_response_id: None,
7313                    })),
7314                ],
7315                _ => vec![
7316                    Ok(InferenceEvent::MessageDelta(MessageDelta {
7317                        text: "final".to_string(),
7318                        phase: None,
7319                    })),
7320                    Ok(InferenceEvent::Completed(CompletionMetadata {
7321                        stop_reason: Some("end_turn".to_string()),
7322                        provider_response_id: None,
7323                    })),
7324                ],
7325            };
7326            Ok(Box::pin(stream::iter(events)))
7327        }
7328    }
7329
7330    #[tokio::test]
7331    async fn turn_completed_reports_terminal_step_finish_reason() {
7332        let mut builder = ExtensionRegistryBuilder::new();
7333        builder.inference_engine(Arc::new(ToolThenStopEngine {
7334            calls: StdMutex::new(0),
7335        }));
7336        builder.tool_contributor(Arc::new(WriteFileContributor));
7337        let runtime = Arc::new(
7338            Runtime::new(
7339                builder.build().unwrap(),
7340                RuntimeConfig {
7341                    policy_mode: PolicyMode::Bypass,
7342                    agent_swarm_mode: false,
7343                    ..RuntimeConfig::default()
7344                },
7345            )
7346            .unwrap(),
7347        );
7348        let mut rx = runtime.subscribe_events();
7349        let turn_id = runtime
7350            .start_turn(StartTurnRequest {
7351                thread_id: "thread-finish-reason".to_string(),
7352                message: "write then finish".to_string(),
7353                images: Vec::new(),
7354                provider_override: None,
7355                model_override: None,
7356                reasoning_override: None,
7357                workspace: test_workspace(),
7358                instructions: InstructionBundle {
7359                    system: None,
7360                    developer: None,
7361                    developer_context: None,
7362                },
7363                developer_context: None,
7364                task_ledger_required: false,
7365            })
7366            .await
7367            .unwrap();
7368
7369        let completed = tokio::time::timeout(std::time::Duration::from_secs(5), async {
7370            loop {
7371                let envelope = rx.recv().await.unwrap();
7372                if envelope.turn_id.as_deref() != Some(&turn_id) {
7373                    continue;
7374                }
7375                match envelope.event {
7376                    RoderEvent::TurnCompleted(event) => break event,
7377                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7378                    _ => {}
7379                }
7380            }
7381        })
7382        .await
7383        .unwrap();
7384
7385        // The mid-turn tool_use stop reason must not leak; the terminal
7386        // end_turn step decides the turn's finish reason.
7387        assert_eq!(completed.finish_reason.as_deref(), Some("stop"));
7388    }
7389
7390    #[tokio::test]
7391    async fn inference_router_selection_changes_request_model_and_records_event() {
7392        let default_requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7393        let routed_requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7394        let contexts = Arc::new(StdMutex::new(Vec::<InferenceRoutingContext>::new()));
7395        let selected = ModelSelection {
7396            provider: "routed-provider".to_string(),
7397            model: "routed-model".to_string(),
7398        };
7399        let default = ModelSelection {
7400            provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7401            model: "mock".to_string(),
7402        };
7403        let decision = InferenceRoutingDecision {
7404            reasoning: Some(ReasoningConfig {
7405                enabled: true,
7406                level: Some(REASONING_LOW.to_string()),
7407            }),
7408            confidence: Some(0.91),
7409            baseline: Some(default.clone()),
7410            matched_signals: vec![roder_api::inference_routing::InferenceRoutingSignal::new(
7411                "intent", "routine",
7412            )],
7413            ..InferenceRoutingDecision::selected("test-router", selected.clone(), "routine request")
7414        };
7415
7416        let mut builder = ExtensionRegistryBuilder::new();
7417        builder.inference_engine(Arc::new(RoutingCaptureEngine {
7418            id: roder_api::catalog::PROVIDER_MOCK,
7419            models: vec![routing_test_model("mock", &[REASONING_LOW])],
7420            requests: default_requests.clone(),
7421        }));
7422        builder.inference_engine(Arc::new(RoutingCaptureEngine {
7423            id: "routed-provider",
7424            models: vec![routing_test_model(
7425                "routed-model",
7426                &[REASONING_LOW, REASONING_MEDIUM],
7427            )],
7428            requests: routed_requests.clone(),
7429        }));
7430        builder.inference_router(Arc::new(StaticRouter {
7431            id: "test-router",
7432            decision,
7433            contexts: contexts.clone(),
7434        }));
7435        let thread_root =
7436            std::env::temp_dir().join(format!("roder-routing-auto-{}", uuid::Uuid::new_v4()));
7437        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
7438            base_path: thread_root.clone(),
7439        }));
7440        let runtime = Arc::new(
7441            Runtime::new(
7442                builder.build().unwrap(),
7443                RuntimeConfig {
7444                    default_provider: default.provider.clone(),
7445                    default_model: default.model.clone(),
7446                    ..RuntimeConfig::default()
7447                },
7448            )
7449            .unwrap(),
7450        );
7451        let thread_id = runtime
7452            .create_thread_with(CreateThreadRequest {
7453                title: Some("Routing auto".to_string()),
7454                workspace: test_workspace(),
7455                workspace_id: None,
7456                root_id: None,
7457                provider: Some(default.provider.clone()),
7458                model: Some(default.model.clone()),
7459                tool_allowlist: Vec::new(),
7460                developer_instructions: None,
7461                external_tools: Vec::new(),
7462                selection_mode: Some(ModelSelectionMode::auto(
7463                    "test-router:coding",
7464                    "test-router",
7465                    "Auto: Coding",
7466                    default.clone(),
7467                    Some("coding".to_string()),
7468                    None,
7469                )),
7470                runner: None,
7471            })
7472            .await
7473            .unwrap()
7474            .thread_id;
7475        let mut rx = runtime.subscribe_events();
7476        let turn_id = runtime
7477            .start_turn(StartTurnRequest {
7478                thread_id: thread_id.clone(),
7479                message: "small cleanup".to_string(),
7480                images: Vec::new(),
7481                provider_override: None,
7482                model_override: None,
7483                reasoning_override: None,
7484                workspace: test_workspace(),
7485                instructions: InstructionBundle::default(),
7486                developer_context: None,
7487                task_ledger_required: false,
7488            })
7489            .await
7490            .unwrap();
7491
7492        let mut routing_event = None;
7493        let mut inference_started = None;
7494        tokio::time::timeout(std::time::Duration::from_secs(5), async {
7495            loop {
7496                let envelope = rx.recv().await.unwrap();
7497                if envelope.turn_id.as_deref() != Some(&turn_id) {
7498                    continue;
7499                }
7500                match envelope.event {
7501                    RoderEvent::InferenceRoutingDecision(event) => {
7502                        routing_event = Some(event);
7503                    }
7504                    RoderEvent::InferenceStarted(event) => {
7505                        inference_started = Some(event);
7506                    }
7507                    RoderEvent::TurnCompleted(_) => break,
7508                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7509                    _ => {}
7510                }
7511            }
7512        })
7513        .await
7514        .unwrap();
7515
7516        assert!(default_requests.lock().unwrap().is_empty());
7517        let routed_requests = routed_requests.lock().unwrap();
7518        assert_eq!(routed_requests.len(), 1);
7519        assert_eq!(routed_requests[0].model, selected);
7520        assert_eq!(
7521            routed_requests[0].reasoning.level.as_deref(),
7522            Some(REASONING_LOW)
7523        );
7524        assert_eq!(
7525            routed_requests[0].metadata["inferenceRouting"]["outcome"],
7526            "selected"
7527        );
7528
7529        let routing_event = routing_event.expect("routing decision event");
7530        assert_eq!(routing_event.default_selection, default);
7531        assert_eq!(routing_event.selected_selection, selected);
7532        assert_eq!(
7533            routing_event.decision.outcome,
7534            InferenceRoutingOutcome::Selected
7535        );
7536        assert_eq!(
7537            inference_started.expect("inference started event").model,
7538            selected
7539        );
7540
7541        let contexts = contexts.lock().unwrap();
7542        assert_eq!(contexts.len(), 1);
7543        assert_eq!(contexts[0].default_selection, default);
7544        assert_eq!(contexts[0].candidates.len(), 2);
7545        assert!(
7546            contexts[0]
7547                .signals
7548                .iter()
7549                .any(|signal| signal.key == "profile" && signal.value == "coding")
7550        );
7551        let _ = std::fs::remove_dir_all(thread_root);
7552    }
7553
7554    #[tokio::test]
7555    async fn inference_router_is_bypassed_for_explicit_selection() {
7556        let requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7557        let contexts = Arc::new(StdMutex::new(Vec::<InferenceRoutingContext>::new()));
7558        let selected = ModelSelection {
7559            provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7560            model: "mock".to_string(),
7561        };
7562
7563        let mut builder = ExtensionRegistryBuilder::new();
7564        builder.inference_engine(Arc::new(RoutingCaptureEngine {
7565            id: roder_api::catalog::PROVIDER_MOCK,
7566            models: vec![routing_test_model("mock", &[REASONING_LOW])],
7567            requests: requests.clone(),
7568        }));
7569        builder.inference_router(Arc::new(StaticRouter {
7570            id: "test-router",
7571            decision: InferenceRoutingDecision::selected(
7572                "test-router",
7573                ModelSelection {
7574                    provider: "missing".to_string(),
7575                    model: "missing".to_string(),
7576                },
7577                "would route if called",
7578            ),
7579            contexts: contexts.clone(),
7580        }));
7581        let thread_root =
7582            std::env::temp_dir().join(format!("roder-routing-explicit-{}", uuid::Uuid::new_v4()));
7583        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
7584            base_path: thread_root.clone(),
7585        }));
7586        let runtime = Arc::new(
7587            Runtime::new(
7588                builder.build().unwrap(),
7589                RuntimeConfig {
7590                    default_provider: selected.provider.clone(),
7591                    default_model: selected.model.clone(),
7592                    inference_router: RuntimeInferenceRouterConfig {
7593                        enabled: true,
7594                        router_id: Some("test-router".to_string()),
7595                    },
7596                    ..RuntimeConfig::default()
7597                },
7598            )
7599            .unwrap(),
7600        );
7601        let thread_id = runtime
7602            .create_thread_with(CreateThreadRequest {
7603                title: Some("Routing explicit".to_string()),
7604                workspace: test_workspace(),
7605                workspace_id: None,
7606                root_id: None,
7607                provider: Some(selected.provider.clone()),
7608                model: Some(selected.model.clone()),
7609                tool_allowlist: Vec::new(),
7610                developer_instructions: None,
7611                external_tools: Vec::new(),
7612                selection_mode: Some(ModelSelectionMode::auto(
7613                    "test-router:default",
7614                    "test-router",
7615                    "Auto",
7616                    selected.clone(),
7617                    None,
7618                    None,
7619                )),
7620                runner: None,
7621            })
7622            .await
7623            .unwrap()
7624            .thread_id;
7625        let mut rx = runtime.subscribe_events();
7626        let turn_id = runtime
7627            .start_turn(StartTurnRequest {
7628                thread_id,
7629                message: "use explicit selection".to_string(),
7630                images: Vec::new(),
7631                provider_override: Some(selected.provider.clone()),
7632                model_override: Some(selected.model.clone()),
7633                reasoning_override: None,
7634                workspace: test_workspace(),
7635                instructions: InstructionBundle::default(),
7636                developer_context: None,
7637                task_ledger_required: false,
7638            })
7639            .await
7640            .unwrap();
7641
7642        let mut saw_routing_event = false;
7643        tokio::time::timeout(std::time::Duration::from_secs(5), async {
7644            loop {
7645                let envelope = rx.recv().await.unwrap();
7646                if envelope.turn_id.as_deref() != Some(&turn_id) {
7647                    continue;
7648                }
7649                match envelope.event {
7650                    RoderEvent::InferenceRoutingDecision(_) => {
7651                        saw_routing_event = true;
7652                    }
7653                    RoderEvent::TurnCompleted(_) => break,
7654                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7655                    _ => {}
7656                }
7657            }
7658        })
7659        .await
7660        .unwrap();
7661
7662        assert!(!saw_routing_event);
7663        assert!(contexts.lock().unwrap().is_empty());
7664        let requests = requests.lock().unwrap();
7665        assert_eq!(requests.len(), 1);
7666        assert_eq!(requests[0].model, selected);
7667        let _ = std::fs::remove_dir_all(thread_root);
7668    }
7669
7670    #[tokio::test]
7671    async fn inference_router_is_bypassed_for_manual_selection_mode() {
7672        let requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7673        let contexts = Arc::new(StdMutex::new(Vec::<InferenceRoutingContext>::new()));
7674        let selected = ModelSelection {
7675            provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7676            model: "mock".to_string(),
7677        };
7678
7679        let mut builder = ExtensionRegistryBuilder::new();
7680        builder.inference_engine(Arc::new(RoutingCaptureEngine {
7681            id: roder_api::catalog::PROVIDER_MOCK,
7682            models: vec![routing_test_model("mock", &[REASONING_LOW])],
7683            requests: requests.clone(),
7684        }));
7685        builder.inference_router(Arc::new(StaticRouter {
7686            id: "test-router",
7687            decision: InferenceRoutingDecision::selected(
7688                "test-router",
7689                ModelSelection {
7690                    provider: "missing".to_string(),
7691                    model: "missing".to_string(),
7692                },
7693                "would route if called",
7694            ),
7695            contexts: contexts.clone(),
7696        }));
7697        let thread_root =
7698            std::env::temp_dir().join(format!("roder-routing-manual-{}", uuid::Uuid::new_v4()));
7699        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
7700            base_path: thread_root.clone(),
7701        }));
7702        let runtime = Arc::new(
7703            Runtime::new(
7704                builder.build().unwrap(),
7705                RuntimeConfig {
7706                    default_provider: selected.provider.clone(),
7707                    default_model: selected.model.clone(),
7708                    inference_router: RuntimeInferenceRouterConfig {
7709                        enabled: true,
7710                        router_id: Some("test-router".to_string()),
7711                    },
7712                    ..RuntimeConfig::default()
7713                },
7714            )
7715            .unwrap(),
7716        );
7717        let thread_id = runtime
7718            .create_thread_with(CreateThreadRequest {
7719                title: Some("Routing manual".to_string()),
7720                workspace: test_workspace(),
7721                workspace_id: None,
7722                root_id: None,
7723                provider: Some(selected.provider.clone()),
7724                model: Some(selected.model.clone()),
7725                tool_allowlist: Vec::new(),
7726                developer_instructions: None,
7727                external_tools: Vec::new(),
7728                selection_mode: Some(ModelSelectionMode::manual(
7729                    selected.provider.clone(),
7730                    selected.model.clone(),
7731                    None,
7732                )),
7733                runner: None,
7734            })
7735            .await
7736            .unwrap()
7737            .thread_id;
7738        let mut rx = runtime.subscribe_events();
7739        let turn_id = runtime
7740            .start_turn(StartTurnRequest {
7741                thread_id,
7742                message: "use selected manual model".to_string(),
7743                images: Vec::new(),
7744                provider_override: None,
7745                model_override: None,
7746                reasoning_override: None,
7747                workspace: test_workspace(),
7748                instructions: InstructionBundle::default(),
7749                developer_context: None,
7750                task_ledger_required: false,
7751            })
7752            .await
7753            .unwrap();
7754
7755        let mut saw_routing_event = false;
7756        tokio::time::timeout(std::time::Duration::from_secs(5), async {
7757            loop {
7758                let envelope = rx.recv().await.unwrap();
7759                if envelope.turn_id.as_deref() != Some(&turn_id) {
7760                    continue;
7761                }
7762                match envelope.event {
7763                    RoderEvent::InferenceRoutingDecision(_) => {
7764                        saw_routing_event = true;
7765                    }
7766                    RoderEvent::TurnCompleted(_) => break,
7767                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7768                    _ => {}
7769                }
7770            }
7771        })
7772        .await
7773        .unwrap();
7774
7775        assert!(!saw_routing_event);
7776        assert!(contexts.lock().unwrap().is_empty());
7777        let requests = requests.lock().unwrap();
7778        assert_eq!(requests.len(), 1);
7779        assert_eq!(requests[0].model, selected);
7780        let _ = std::fs::remove_dir_all(thread_root);
7781    }
7782
7783    #[test]
7784    fn enabled_inference_router_requires_registered_router() {
7785        let mut builder = ExtensionRegistryBuilder::new();
7786        builder.inference_engine(Arc::new(FakeInferenceEngine));
7787
7788        let err = match Runtime::new(
7789            builder.build().unwrap(),
7790            RuntimeConfig {
7791                inference_router: RuntimeInferenceRouterConfig {
7792                    enabled: true,
7793                    router_id: Some("missing-router".to_string()),
7794                },
7795                ..RuntimeConfig::default()
7796            },
7797        ) {
7798            Ok(_) => panic!("runtime should reject unknown inference router"),
7799            Err(err) => err,
7800        };
7801
7802        assert!(
7803            err.to_string()
7804                .contains("inference router \"missing-router\" is not registered")
7805        );
7806    }
7807
7808    #[tokio::test]
7809    async fn model_profile_routes_request_knobs_to_next_inference() {
7810        let request = captured_profile_request(RuntimeConfig {
7811            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7812            default_model: "gpt-5.5".to_string(),
7813            model_profiles: std::collections::HashMap::from([(
7814                "gpt-5.5".to_string(),
7815                test_model_profile("gpt-5.5"),
7816            )]),
7817            ..RuntimeConfig::default()
7818        })
7819        .await;
7820
7821        let tool_names = request
7822            .tools
7823            .iter()
7824            .map(|tool| tool.name.as_str())
7825            .collect::<Vec<_>>();
7826        assert!(tool_names.contains(&"apply_patch"));
7827        assert!(tool_names.contains(&"edit"));
7828        assert!(tool_names.contains(&"multi_edit"));
7829        assert!(tool_names.contains(&"write_file"));
7830        assert_eq!(request.reasoning.level.as_deref(), Some(REASONING_LOW));
7831        assert_eq!(request.runtime.parallel_tool_calls, Some(false));
7832        assert_eq!(request.runtime.auto_compact_token_limit, Some(123_000));
7833        assert!(
7834            request
7835                .instructions
7836                .developer
7837                .as_deref()
7838                .unwrap_or_default()
7839                .contains("Use the provided context as the current working set")
7840        );
7841        assert_eq!(
7842            request
7843                .metadata
7844                .pointer("/modelProfile/schemaPolicy")
7845                .and_then(serde_json::Value::as_str),
7846            Some("standard_required_first")
7847        );
7848    }
7849
7850    #[tokio::test]
7851    async fn ultra_reasoning_reaches_transport_state_and_enables_proactive_delegation() {
7852        let request = captured_profile_request(RuntimeConfig {
7853            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7854            default_model: "gpt-5.6-sol".to_string(),
7855            reasoning: Some(REASONING_ULTRA.to_string()),
7856            ..RuntimeConfig::default()
7857        })
7858        .await;
7859
7860        assert_eq!(request.reasoning.level.as_deref(), Some(REASONING_ULTRA));
7861        let developer = request
7862            .instructions
7863            .developer
7864            .as_deref()
7865            .expect("ultra developer instructions");
7866        assert!(developer.contains("Proactive multi-agent delegation is active"));
7867    }
7868
7869    #[tokio::test]
7870    async fn lower_sol_effort_requires_explicit_multi_agent_request() {
7871        let request = captured_profile_request(RuntimeConfig {
7872            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7873            default_model: "gpt-5.6-sol".to_string(),
7874            reasoning: Some(roder_api::catalog::REASONING_MEDIUM.to_string()),
7875            ..RuntimeConfig::default()
7876        })
7877        .await;
7878
7879        let developer = request
7880            .instructions
7881            .developer
7882            .as_deref()
7883            .expect("sol developer instructions");
7884        assert!(developer.contains("Do not spawn sub-agents unless"));
7885        assert!(!developer.contains("Proactive multi-agent delegation is active"));
7886    }
7887
7888    #[tokio::test]
7889    async fn luna_does_not_receive_codex_v2_multi_agent_policy() {
7890        let request = captured_profile_request(RuntimeConfig {
7891            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7892            default_model: "gpt-5.6-luna".to_string(),
7893            reasoning: Some(roder_api::catalog::REASONING_MAX.to_string()),
7894            ..RuntimeConfig::default()
7895        })
7896        .await;
7897
7898        let developer = request
7899            .instructions
7900            .developer
7901            .as_deref()
7902            .unwrap_or_default();
7903        assert!(!developer.contains("Do not spawn sub-agents unless"));
7904        assert!(!developer.contains("Proactive multi-agent delegation is active"));
7905    }
7906
7907    #[tokio::test]
7908    async fn turn_developer_context_reaches_inference_and_does_not_persist() {
7909        let captured = Arc::new(StdMutex::new(None));
7910        let mut builder = ExtensionRegistryBuilder::new();
7911        builder.inference_engine(Arc::new(CapturingEngine {
7912            request: captured.clone(),
7913        }));
7914        let runtime = Arc::new(
7915            Runtime::new(
7916                builder.build().unwrap(),
7917                RuntimeConfig {
7918                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7919                    default_model: "gpt-5.5".to_string(),
7920                    ..RuntimeConfig::default()
7921                },
7922            )
7923            .unwrap(),
7924        );
7925
7926        async fn run_turn(runtime: &Arc<Runtime>, developer_context: Option<String>) {
7927            let mut rx = runtime.subscribe_events();
7928            let turn_id = runtime
7929                .start_turn(StartTurnRequest {
7930                    thread_id: "thread-turn-context".to_string(),
7931                    message: "hello".to_string(),
7932                    images: Vec::new(),
7933                    provider_override: None,
7934                    model_override: None,
7935                    reasoning_override: None,
7936                    workspace: test_workspace(),
7937                    instructions: InstructionBundle::default(),
7938                    developer_context,
7939                    task_ledger_required: false,
7940                })
7941                .await
7942                .unwrap();
7943            tokio::time::timeout(std::time::Duration::from_secs(5), async {
7944                loop {
7945                    let envelope = rx.recv().await.unwrap();
7946                    if envelope.turn_id.as_deref() != Some(&turn_id) {
7947                        continue;
7948                    }
7949                    match envelope.event {
7950                        RoderEvent::TurnCompleted(_) => break,
7951                        RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7952                        _ => {}
7953                    }
7954                }
7955            })
7956            .await
7957            .unwrap();
7958        }
7959
7960        run_turn(
7961            &runtime,
7962            Some("Connected accounts: example-service.".to_string()),
7963        )
7964        .await;
7965        let request = captured.lock().unwrap().clone().unwrap();
7966        assert_eq!(
7967            request.instructions.developer_context.as_deref(),
7968            Some("Connected accounts: example-service.")
7969        );
7970
7971        // The context is per-turn only: the next turn on the same thread
7972        // without a developerContext must not see the previous one.
7973        run_turn(&runtime, None).await;
7974        let request = captured.lock().unwrap().clone().unwrap();
7975        assert_eq!(request.instructions.developer_context, None);
7976    }
7977
7978    #[tokio::test]
7979    async fn tool_search_overrides_route_to_next_inference_request() {
7980        let request = captured_profile_request(RuntimeConfig {
7981            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7982            default_model: "gpt-5.4".to_string(),
7983            tool_search: ToolSearchConfig {
7984                mode: roder_api::inference::ToolSearchMode::Auto,
7985                max_catalog_items: Some(100),
7986                ..ToolSearchConfig::default()
7987            },
7988            provider_tool_search: std::collections::HashMap::from([(
7989                roder_api::catalog::PROVIDER_MOCK.to_string(),
7990                roder_api::inference::ToolSearchConfigOverlay {
7991                    include_skills: Some(false),
7992                    provider_variant: Some(roder_api::inference::ToolSearchProviderVariant::Regex),
7993                    ..Default::default()
7994                },
7995            )]),
7996            model_tool_search: std::collections::HashMap::from([(
7997                "gpt-5.4".to_string(),
7998                roder_api::inference::ToolSearchConfigOverlay {
7999                    mode: Some(roder_api::inference::ToolSearchMode::ProviderNative),
8000                    max_catalog_items: Some(25),
8001                    provider_variant: Some(roder_api::inference::ToolSearchProviderVariant::Bm25),
8002                    ..Default::default()
8003                },
8004            )]),
8005            ..RuntimeConfig::default()
8006        })
8007        .await;
8008
8009        assert_eq!(
8010            request.runtime.tool_search.mode,
8011            roder_api::inference::ToolSearchMode::ProviderNative
8012        );
8013        assert_eq!(request.runtime.tool_search.max_catalog_items, Some(25));
8014        assert_eq!(
8015            request.runtime.tool_search.provider_variant,
8016            roder_api::inference::ToolSearchProviderVariant::Bm25
8017        );
8018    }
8019
8020    #[tokio::test]
8021    async fn context_entrypoint_hints_use_turn_workspace() {
8022        let process_workspace = runtime_test_workspace("entrypoint-process");
8023        let thread_workspace = runtime_test_workspace("entrypoint-thread");
8024        std::fs::create_dir_all(process_workspace.join("src")).unwrap();
8025        std::fs::create_dir_all(thread_workspace.join("src")).unwrap();
8026        std::fs::write(
8027            process_workspace.join("src/sidebar-thread-groups.ts"),
8028            "export const desktopLeak = true;\n",
8029        )
8030        .unwrap();
8031        std::fs::write(
8032            thread_workspace.join("src/voice-plan-feedback.ts"),
8033            "export const voicePlanFeedback = true;\n",
8034        )
8035        .unwrap();
8036
8037        let captured = Arc::new(StdMutex::new(None));
8038        let mut builder = ExtensionRegistryBuilder::new();
8039        builder.inference_engine(Arc::new(CapturingEngine {
8040            request: captured.clone(),
8041        }));
8042        builder.context_planner(Arc::new(roder_context::EntrypointContextPlanner::new(
8043            process_workspace.clone(),
8044        )));
8045        let runtime = Arc::new(
8046            Runtime::new(
8047                builder.build().unwrap(),
8048                RuntimeConfig {
8049                    workspace: Some(process_workspace.display().to_string()),
8050                    ..RuntimeConfig::default()
8051                },
8052            )
8053            .unwrap(),
8054        );
8055        let mut rx = runtime.subscribe_events();
8056
8057        let turn_id = runtime
8058            .start_turn(StartTurnRequest {
8059                thread_id: "thread-workspace-entrypoint".to_string(),
8060                message: "investigate voice plan feedback".to_string(),
8061                images: Vec::new(),
8062                provider_override: None,
8063                model_override: None,
8064                reasoning_override: None,
8065                workspace: thread_workspace.display().to_string(),
8066                instructions: crate::instructions::default_instructions(),
8067                developer_context: None,
8068                task_ledger_required: false,
8069            })
8070            .await
8071            .unwrap();
8072
8073        tokio::time::timeout(std::time::Duration::from_secs(5), async {
8074            loop {
8075                let envelope = rx.recv().await.unwrap();
8076                if envelope.turn_id.as_deref() != Some(&turn_id) {
8077                    continue;
8078                }
8079                match envelope.event {
8080                    RoderEvent::TurnCompleted(_) => break,
8081                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8082                    _ => {}
8083                }
8084            }
8085        })
8086        .await
8087        .unwrap();
8088
8089        let request = captured.lock().unwrap().clone().expect("captured request");
8090        let transcript_text = request
8091            .transcript
8092            .iter()
8093            .map(|item| match item {
8094                TranscriptItem::UserMessage(message) => message.text.as_str(),
8095                _ => "",
8096            })
8097            .collect::<Vec<_>>()
8098            .join("\n");
8099        assert!(transcript_text.contains("src/voice-plan-feedback.ts"));
8100        assert!(!transcript_text.contains("src/sidebar-thread-groups.ts"));
8101
8102        let _ = std::fs::remove_dir_all(process_workspace);
8103        let _ = std::fs::remove_dir_all(thread_workspace);
8104    }
8105
8106    fn runtime_test_workspace(name: &str) -> std::path::PathBuf {
8107        let path =
8108            std::env::temp_dir().join(format!("roder-runtime-{name}-{}", uuid::Uuid::new_v4()));
8109        let _ = std::fs::remove_dir_all(&path);
8110        std::fs::create_dir_all(&path).unwrap();
8111        path
8112    }
8113
8114    #[tokio::test]
8115    async fn model_profile_user_model_knobs_override_profile_defaults() {
8116        let request = captured_profile_request(RuntimeConfig {
8117            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8118            default_model: "gpt-5.5".to_string(),
8119            reasoning: Some(REASONING_HIGH.to_string()),
8120            auto_compact_token_limit: Some(999),
8121            model_edit_tools: std::collections::HashMap::from([(
8122                "gpt-5.5".to_string(),
8123                EDIT_TOOL_PATCH.to_string(),
8124            )]),
8125            model_parallel_tool_calls: std::collections::HashMap::from([(
8126                "gpt-5.5".to_string(),
8127                true,
8128            )]),
8129            model_profiles: std::collections::HashMap::from([(
8130                "gpt-5.5".to_string(),
8131                test_model_profile("gpt-5.5"),
8132            )]),
8133            ..RuntimeConfig::default()
8134        })
8135        .await;
8136
8137        let tool_names = request
8138            .tools
8139            .iter()
8140            .map(|tool| tool.name.as_str())
8141            .collect::<Vec<_>>();
8142        assert!(tool_names.contains(&"apply_patch"));
8143        assert!(!tool_names.contains(&"edit"));
8144        assert_eq!(request.reasoning.level.as_deref(), Some(REASONING_HIGH));
8145        assert_eq!(request.runtime.parallel_tool_calls, Some(true));
8146        assert_eq!(request.runtime.auto_compact_token_limit, Some(999));
8147    }
8148
8149    #[tokio::test]
8150    async fn runtime_tool_allowlist_filters_advertised_tools() {
8151        let request = captured_profile_request(RuntimeConfig {
8152            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8153            default_model: "gpt-5.5".to_string(),
8154            tool_allowlist: vec!["edit".to_string()],
8155            model_profiles: std::collections::HashMap::from([(
8156                "gpt-5.5".to_string(),
8157                test_model_profile("gpt-5.5"),
8158            )]),
8159            ..RuntimeConfig::default()
8160        })
8161        .await;
8162
8163        let tool_names = request
8164            .tools
8165            .iter()
8166            .map(|tool| tool.name.as_str())
8167            .collect::<Vec<_>>();
8168        assert_eq!(tool_names, vec!["edit"]);
8169    }
8170
8171    /// Builds a store-backed runtime turn for a thread created with the given per-thread
8172    /// overrides and returns the captured inference request.
8173    async fn captured_thread_override_request(
8174        runtime: &Arc<Runtime>,
8175        requests: &Arc<StdMutex<Vec<AgentInferenceRequest>>>,
8176        tool_allowlist: Vec<String>,
8177        developer_instructions: Option<String>,
8178        external_tools: Vec<ToolSpec>,
8179    ) -> AgentInferenceRequest {
8180        let thread_id = runtime
8181            .create_thread_with(CreateThreadRequest {
8182                title: Some("Thread overrides".to_string()),
8183                workspace: test_workspace(),
8184                workspace_id: None,
8185                root_id: None,
8186                provider: None,
8187                model: None,
8188                selection_mode: None,
8189                tool_allowlist,
8190                developer_instructions,
8191                external_tools,
8192                runner: None,
8193            })
8194            .await
8195            .unwrap()
8196            .thread_id;
8197        let mut rx = runtime.subscribe_events();
8198        let turn_id = runtime
8199            .start_turn(StartTurnRequest {
8200                thread_id,
8201                message: "hello".to_string(),
8202                images: Vec::new(),
8203                provider_override: None,
8204                model_override: None,
8205                reasoning_override: None,
8206                workspace: test_workspace(),
8207                instructions: crate::default_instructions(),
8208                developer_context: None,
8209                task_ledger_required: false,
8210            })
8211            .await
8212            .unwrap();
8213        tokio::time::timeout(std::time::Duration::from_secs(5), async {
8214            loop {
8215                let envelope = rx.recv().await.unwrap();
8216                if envelope.turn_id.as_deref() != Some(&turn_id) {
8217                    continue;
8218                }
8219                match envelope.event {
8220                    RoderEvent::TurnCompleted(_) => break,
8221                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8222                    _ => {}
8223                }
8224            }
8225        })
8226        .await
8227        .unwrap();
8228        requests.lock().unwrap().pop().expect("captured request")
8229    }
8230
8231    #[tokio::test]
8232    async fn thread_tool_allowlist_filters_only_that_thread() {
8233        let requests = Arc::new(StdMutex::new(Vec::new()));
8234        let thread_root =
8235            std::env::temp_dir().join(format!("roder-thread-allowlist-{}", uuid::Uuid::new_v4()));
8236        let mut builder = ExtensionRegistryBuilder::new();
8237        builder.inference_engine(Arc::new(SwitchCaptureEngine {
8238            requests: requests.clone(),
8239        }));
8240        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8241            base_path: thread_root.clone(),
8242        }));
8243        builder.tool_contributor(Arc::new(ProfileToolContributor));
8244        let runtime = Arc::new(
8245            Runtime::new(
8246                builder.build().unwrap(),
8247                RuntimeConfig {
8248                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8249                    default_model: "gpt-5.5".to_string(),
8250                    model_profiles: std::collections::HashMap::from([(
8251                        "gpt-5.5".to_string(),
8252                        test_model_profile("gpt-5.5"),
8253                    )]),
8254                    ..RuntimeConfig::default()
8255                },
8256            )
8257            .unwrap(),
8258        );
8259
8260        let allowlisted = captured_thread_override_request(
8261            &runtime,
8262            &requests,
8263            vec!["edit".to_string()],
8264            None,
8265            Vec::new(),
8266        )
8267        .await;
8268        let unrestricted =
8269            captured_thread_override_request(&runtime, &requests, Vec::new(), None, Vec::new())
8270                .await;
8271
8272        let allowlisted_names = allowlisted
8273            .tools
8274            .iter()
8275            .map(|tool| tool.name.as_str())
8276            .collect::<Vec<_>>();
8277        assert_eq!(allowlisted_names, vec!["edit"]);
8278        let unrestricted_names = unrestricted
8279            .tools
8280            .iter()
8281            .map(|tool| tool.name.as_str())
8282            .collect::<Vec<_>>();
8283        assert!(unrestricted_names.contains(&"edit"));
8284        assert!(unrestricted_names.len() > 1);
8285
8286        let _ = std::fs::remove_dir_all(thread_root);
8287    }
8288
8289    /// Builds a store-backed runtime whose `RuntimeConfig.tool_allowlist` is `["edit"]`.
8290    fn runtime_with_edit_allowlist(
8291        requests: &Arc<StdMutex<Vec<AgentInferenceRequest>>>,
8292        thread_root: &std::path::Path,
8293    ) -> Arc<Runtime> {
8294        let mut builder = ExtensionRegistryBuilder::new();
8295        builder.inference_engine(Arc::new(SwitchCaptureEngine {
8296            requests: requests.clone(),
8297        }));
8298        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8299            base_path: thread_root.to_path_buf(),
8300        }));
8301        builder.tool_contributor(Arc::new(ProfileToolContributor));
8302        Arc::new(
8303            Runtime::new(
8304                builder.build().unwrap(),
8305                RuntimeConfig {
8306                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8307                    default_model: "gpt-5.5".to_string(),
8308                    tool_allowlist: vec!["edit".to_string()],
8309                    model_profiles: std::collections::HashMap::from([(
8310                        "gpt-5.5".to_string(),
8311                        test_model_profile("gpt-5.5"),
8312                    )]),
8313                    ..RuntimeConfig::default()
8314                },
8315            )
8316            .unwrap(),
8317        )
8318    }
8319
8320    #[tokio::test]
8321    async fn runtime_and_thread_allowlists_intersect() {
8322        let requests = Arc::new(StdMutex::new(Vec::new()));
8323        let thread_root = std::env::temp_dir().join(format!(
8324            "roder-allowlist-intersect-{}",
8325            uuid::Uuid::new_v4()
8326        ));
8327        let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8328
8329        let request = captured_thread_override_request(
8330            &runtime,
8331            &requests,
8332            vec!["edit".to_string(), "write_file".to_string()],
8333            None,
8334            Vec::new(),
8335        )
8336        .await;
8337
8338        // The thread allowlist must not re-enable tools the runtime allowlist bans.
8339        let names = request
8340            .tools
8341            .iter()
8342            .map(|tool| tool.name.as_str())
8343            .collect::<Vec<_>>();
8344        assert_eq!(names, vec!["edit"]);
8345
8346        let _ = std::fs::remove_dir_all(thread_root);
8347    }
8348
8349    #[tokio::test]
8350    async fn route_tool_call_denies_tools_outside_allowlists() {
8351        let requests = Arc::new(StdMutex::new(Vec::new()));
8352        let thread_root =
8353            std::env::temp_dir().join(format!("roder-allowlist-dispatch-{}", uuid::Uuid::new_v4()));
8354        let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8355        let thread_id = runtime
8356            .create_thread_with(CreateThreadRequest {
8357                title: Some("Dispatch allowlist".to_string()),
8358                workspace: test_workspace(),
8359                workspace_id: None,
8360                root_id: None,
8361                provider: None,
8362                model: None,
8363                selection_mode: None,
8364                tool_allowlist: vec!["edit".to_string(), "write_file".to_string()],
8365                developer_instructions: None,
8366                external_tools: Vec::new(),
8367                runner: None,
8368            })
8369            .await
8370            .unwrap()
8371            .thread_id;
8372
8373        // write_file is registered and on the thread allowlist but banned by the runtime allowlist.
8374        let result = runtime
8375            .route_tool_call(
8376                &thread_id,
8377                &"turn-allowlist-dispatch".to_string(),
8378                roder_api::inference::ToolCallCompleted {
8379                    id: "call-1".to_string(),
8380                    name: "write_file".to_string(),
8381                    arguments: r#"{"path":"a.txt","content":"hi"}"#.to_string(),
8382                },
8383                None,
8384                None,
8385            )
8386            .await
8387            .unwrap();
8388
8389        assert!(result.is_error);
8390        assert!(
8391            result
8392                .result
8393                .contains("not permitted by the tool allowlist"),
8394            "unexpected result: {}",
8395            result.result
8396        );
8397
8398        let _ = std::fs::remove_dir_all(thread_root);
8399    }
8400
8401    #[tokio::test]
8402    async fn route_tool_calls_denies_agent_swarm_mixed_with_other_tools() {
8403        let requests = Arc::new(StdMutex::new(Vec::new()));
8404        let thread_root =
8405            std::env::temp_dir().join(format!("roder-swarm-exclusive-{}", uuid::Uuid::new_v4()));
8406        let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8407
8408        // A response that mixes agent_swarm with another tool is denied wholesale:
8409        // both calls get an error result with retry guidance, and no tool runs.
8410        let results = runtime
8411            .route_tool_calls(
8412                &"thread-swarm".to_string(),
8413                &"turn-swarm".to_string(),
8414                vec![
8415                    roder_api::inference::ToolCallCompleted {
8416                        id: "swarm-1".to_string(),
8417                        name: "agent_swarm".to_string(),
8418                        arguments: "{}".to_string(),
8419                    },
8420                    roder_api::inference::ToolCallCompleted {
8421                        id: "read-1".to_string(),
8422                        name: "read_file".to_string(),
8423                        arguments: "{}".to_string(),
8424                    },
8425                ],
8426                true,
8427                None,
8428                None,
8429            )
8430            .await
8431            .unwrap();
8432
8433        assert_eq!(results.len(), 2, "every tool_call_id must get a response");
8434        assert_eq!(results[0].id, "swarm-1");
8435        assert_eq!(results[1].id, "read-1");
8436        for result in &results {
8437            assert!(result.is_error);
8438            assert!(
8439                result.result.contains("only tool call"),
8440                "unexpected result: {}",
8441                result.result
8442            );
8443        }
8444
8445        let _ = std::fs::remove_dir_all(thread_root);
8446    }
8447
8448    #[tokio::test]
8449    async fn route_tool_calls_emits_agent_swarm_started_event() {
8450        let requests = Arc::new(StdMutex::new(Vec::new()));
8451        let thread_root =
8452            std::env::temp_dir().join(format!("roder-swarm-started-{}", uuid::Uuid::new_v4()));
8453        let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8454        let mut events = runtime.subscribe_events();
8455
8456        // A single agent_swarm call (the valid shape) emits AgentSwarmStarted on
8457        // the event bus before dispatch, with the child count parsed from args.
8458        let _ = runtime
8459            .route_tool_calls(
8460                &"thread-swarm".to_string(),
8461                &"turn-swarm".to_string(),
8462                vec![roder_api::inference::ToolCallCompleted {
8463                    id: "swarm-1".to_string(),
8464                    name: "agent_swarm".to_string(),
8465                    arguments: r#"{"description":"x","prompt_template":"Read {{item}}","items":["a.rs","b.rs"]}"#
8466                        .to_string(),
8467                }],
8468                true,
8469                None,
8470                None,
8471            )
8472            .await
8473            .unwrap();
8474
8475        let mut started_child_count = None;
8476        for _ in 0..16 {
8477            let envelope = tokio::time::timeout(std::time::Duration::from_secs(2), events.recv())
8478                .await
8479                .unwrap()
8480                .unwrap();
8481            if let RoderEvent::AgentSwarmStarted(event) = envelope.event {
8482                started_child_count = Some(event.child_count);
8483                assert_eq!(event.tool_id, "swarm-1");
8484                break;
8485            }
8486        }
8487        assert_eq!(started_child_count, Some(2));
8488
8489        let _ = std::fs::remove_dir_all(thread_root);
8490    }
8491
8492    #[test]
8493    fn agent_swarm_child_count_sums_items_and_resumes() {
8494        assert_eq!(
8495            agent_swarm_child_count(r#"{"description":"x","items":["a","b","c"]}"#),
8496            3
8497        );
8498        assert_eq!(
8499            agent_swarm_child_count(
8500                r#"{"description":"x","items":["a"],"resume_agent_ids":{"id1":"continue"}}"#
8501            ),
8502            2
8503        );
8504        // Malformed input is lenient.
8505        assert_eq!(agent_swarm_child_count("not json"), 0);
8506    }
8507
8508    #[test]
8509    fn parse_swarm_counts_reads_summary_with_omitted_buckets() {
8510        let text = "<agent_swarm_result>\n<summary>completed: 2, failed: 1</summary>\n</agent_swarm_result>";
8511        assert_eq!(parse_swarm_counts(text), Some((2, 1, 0)));
8512        let text = "<agent_swarm_result>\n<summary>completed: 0</summary>\n</agent_swarm_result>";
8513        assert_eq!(parse_swarm_counts(text), Some((0, 0, 0)));
8514        // Not a swarm result.
8515        assert_eq!(parse_swarm_counts("just text"), None);
8516    }
8517
8518    /// Signals when inference starts, then waits for `proceed` and fails the stream.
8519    struct SignalledFailureEngine {
8520        started: tokio::sync::mpsc::UnboundedSender<()>,
8521        proceed: Arc<tokio::sync::Notify>,
8522    }
8523
8524    #[async_trait::async_trait]
8525    impl InferenceEngine for SignalledFailureEngine {
8526        fn id(&self) -> String {
8527            roder_api::catalog::PROVIDER_MOCK.to_string()
8528        }
8529
8530        fn capabilities(&self) -> InferenceCapabilities {
8531            InferenceCapabilities::coding_agent_default()
8532        }
8533
8534        async fn list_models(
8535            &self,
8536            _ctx: InferenceProviderContext<'_>,
8537        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
8538            Ok(roder_api::catalog::models_for_provider(
8539                roder_api::catalog::PROVIDER_MOCK,
8540                true,
8541            ))
8542        }
8543
8544        async fn stream_turn(
8545            &self,
8546            _ctx: InferenceTurnContext<'_>,
8547            _request: AgentInferenceRequest,
8548        ) -> anyhow::Result<InferenceEventStream> {
8549            let _ = self.started.send(());
8550            self.proceed.notified().await;
8551            anyhow::bail!("engine failed mid-turn")
8552        }
8553    }
8554
8555    #[tokio::test]
8556    async fn failed_turn_sweeps_pending_external_tool_calls() {
8557        let (started_tx, mut started_rx) = tokio::sync::mpsc::unbounded_channel();
8558        let proceed = Arc::new(tokio::sync::Notify::new());
8559        let mut builder = ExtensionRegistryBuilder::new();
8560        builder.inference_engine(Arc::new(SignalledFailureEngine {
8561            started: started_tx,
8562            proceed: proceed.clone(),
8563        }));
8564        let runtime =
8565            Arc::new(Runtime::new(builder.build().unwrap(), RuntimeConfig::default()).unwrap());
8566        let mut rx = runtime.subscribe_events();
8567        let turn_id = runtime
8568            .start_turn(StartTurnRequest {
8569                thread_id: "thread-sweep".to_string(),
8570                message: "go".to_string(),
8571                images: Vec::new(),
8572                provider_override: None,
8573                model_override: None,
8574                reasoning_override: None,
8575                workspace: test_workspace(),
8576                instructions: InstructionBundle {
8577                    system: None,
8578                    developer: None,
8579                    developer_context: None,
8580                },
8581                developer_context: None,
8582                task_ledger_required: false,
8583            })
8584            .await
8585            .unwrap();
8586        tokio::time::timeout(std::time::Duration::from_secs(5), started_rx.recv())
8587            .await
8588            .unwrap()
8589            .unwrap();
8590
8591        let (tx, _pending_rx) = oneshot::channel();
8592        runtime.pending_external_tool_calls.lock().await.insert(
8593            "exttool-sweep-test".to_string(),
8594            PendingExternalToolCall {
8595                thread_id: "thread-sweep".to_string(),
8596                turn_id: turn_id.clone(),
8597                tool_id: "call-1".to_string(),
8598                tool_name: "acme_lookup".to_string(),
8599                tx,
8600            },
8601        );
8602        proceed.notify_one();
8603
8604        let outcome = tokio::time::timeout(std::time::Duration::from_secs(5), async {
8605            loop {
8606                let envelope = rx.recv().await.unwrap();
8607                if let RoderEvent::ExternalToolCallResolved(event) = envelope.event
8608                    && event.request_id == "exttool-sweep-test"
8609                {
8610                    break event.outcome;
8611                }
8612            }
8613        })
8614        .await
8615        .expect("turn failure must resolve pending external tool calls");
8616        assert_eq!(outcome, ExternalToolCallOutcome::Cancelled);
8617        assert!(runtime.pending_external_tool_calls.lock().await.is_empty());
8618    }
8619
8620    #[tokio::test]
8621    async fn thread_developer_instructions_layer_under_harness_prompt() {
8622        let requests = Arc::new(StdMutex::new(Vec::new()));
8623        let thread_root = std::env::temp_dir().join(format!(
8624            "roder-thread-instructions-{}",
8625            uuid::Uuid::new_v4()
8626        ));
8627        let mut builder = ExtensionRegistryBuilder::new();
8628        builder.inference_engine(Arc::new(SwitchCaptureEngine {
8629            requests: requests.clone(),
8630        }));
8631        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8632            base_path: thread_root.clone(),
8633        }));
8634        builder.tool_contributor(Arc::new(ProfileToolContributor));
8635        let runtime =
8636            Arc::new(Runtime::new(builder.build().unwrap(), RuntimeConfig::default()).unwrap());
8637
8638        let request = captured_thread_override_request(
8639            &runtime,
8640            &requests,
8641            Vec::new(),
8642            Some("You are embedded in a host app.".to_string()),
8643            Vec::new(),
8644        )
8645        .await;
8646
8647        let system = request.instructions.system.expect("system instructions");
8648        assert!(system.starts_with("You are Roder"));
8649        let developer = request
8650            .instructions
8651            .developer
8652            .expect("developer instructions");
8653        assert!(developer.starts_with("You are embedded in a host app."));
8654
8655        let plain =
8656            captured_thread_override_request(&runtime, &requests, Vec::new(), None, Vec::new())
8657                .await;
8658        assert_eq!(plain.instructions.developer, None);
8659
8660        let _ = std::fs::remove_dir_all(thread_root);
8661    }
8662
8663    #[tokio::test]
8664    async fn thread_external_tools_are_advertised_and_shadow_builtins() {
8665        let requests = Arc::new(StdMutex::new(Vec::new()));
8666        let thread_root = std::env::temp_dir().join(format!(
8667            "roder-thread-external-tools-{}",
8668            uuid::Uuid::new_v4()
8669        ));
8670        let mut builder = ExtensionRegistryBuilder::new();
8671        builder.inference_engine(Arc::new(SwitchCaptureEngine {
8672            requests: requests.clone(),
8673        }));
8674        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8675            base_path: thread_root.clone(),
8676        }));
8677        builder.tool_contributor(Arc::new(ProfileToolContributor));
8678        let runtime =
8679            Arc::new(Runtime::new(builder.build().unwrap(), RuntimeConfig::default()).unwrap());
8680
8681        let external_tools = vec![
8682            ToolSpec {
8683                name: "acme_lookup".to_string(),
8684                description: "Look up Acme workspace state.".to_string(),
8685                parameters: serde_json::json!({
8686                    "type": "object",
8687                    "properties": { "query": { "type": "string" } },
8688                    "required": ["query"]
8689                }),
8690            },
8691            ToolSpec {
8692                name: "edit".to_string(),
8693                description: "Host-managed edit.".to_string(),
8694                parameters: serde_json::json!({ "type": "object" }),
8695            },
8696        ];
8697        let request =
8698            captured_thread_override_request(&runtime, &requests, Vec::new(), None, external_tools)
8699                .await;
8700
8701        let acme = request
8702            .tools
8703            .iter()
8704            .find(|tool| tool.name == "acme_lookup")
8705            .expect("external tool advertised");
8706        assert_eq!(acme.description, "Look up Acme workspace state.");
8707        assert_eq!(acme.parameters["required"][0], "query");
8708        let edits = request
8709            .tools
8710            .iter()
8711            .filter(|tool| tool.name == "edit")
8712            .collect::<Vec<_>>();
8713        assert_eq!(edits.len(), 1, "external edit shadows the builtin");
8714        assert_eq!(edits[0].description, "Host-managed edit.");
8715
8716        let plain =
8717            captured_thread_override_request(&runtime, &requests, Vec::new(), None, Vec::new())
8718                .await;
8719        assert!(plain.tools.iter().all(|tool| tool.name != "acme_lookup"));
8720        let plain_edit = plain
8721            .tools
8722            .iter()
8723            .find(|tool| tool.name == "edit")
8724            .expect("builtin edit advertised on plain thread");
8725        assert_eq!(plain_edit.description, "edit test tool");
8726
8727        let _ = std::fs::remove_dir_all(thread_root);
8728    }
8729
8730    #[tokio::test]
8731    async fn model_switch_injects_summary_and_records_profile_segments() {
8732        let requests = Arc::new(StdMutex::new(Vec::new()));
8733        let thread_root = std::env::temp_dir().join(format!(
8734            "roder-model-switch-thread-{}",
8735            uuid::Uuid::new_v4()
8736        ));
8737        let mut builder = ExtensionRegistryBuilder::new();
8738        builder.inference_engine(Arc::new(SwitchCaptureEngine {
8739            requests: requests.clone(),
8740        }));
8741        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8742            base_path: thread_root.clone(),
8743        }));
8744        builder.tool_contributor(Arc::new(ProfileToolContributor));
8745        let mut claude_profile = test_model_profile("claude-haiku-4-5-20251001");
8746        claude_profile.provider_family = ProviderFamily::Anthropic;
8747        claude_profile.edit_tool = Some(EDIT_TOOL_EDIT.to_string());
8748        let runtime = Arc::new(
8749            Runtime::new(
8750                builder.build().unwrap(),
8751                RuntimeConfig {
8752                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8753                    default_model: "gpt-5.5".to_string(),
8754                    model_profiles: std::collections::HashMap::from([
8755                        ("gpt-5.5".to_string(), test_model_profile("gpt-5.5")),
8756                        ("claude-haiku-4-5-20251001".to_string(), claude_profile),
8757                    ]),
8758                    ..RuntimeConfig::default()
8759                },
8760            )
8761            .unwrap(),
8762        );
8763        let thread_id = runtime
8764            .create_thread_with(CreateThreadRequest {
8765                title: Some("Model switch".to_string()),
8766                workspace: test_workspace(),
8767                workspace_id: None,
8768                root_id: None,
8769                provider: None,
8770                model: None,
8771                selection_mode: None,
8772                tool_allowlist: Vec::new(),
8773                developer_instructions: None,
8774                external_tools: Vec::new(),
8775                runner: None,
8776            })
8777            .await
8778            .unwrap()
8779            .thread_id;
8780        let mut rx = runtime.subscribe_events();
8781        for (message, model_override) in [
8782            ("first turn", None),
8783            ("second turn", Some("claude-haiku-4-5-20251001".to_string())),
8784        ] {
8785            let turn_id = runtime
8786                .start_turn(StartTurnRequest {
8787                    thread_id: thread_id.clone(),
8788                    message: message.to_string(),
8789                    images: Vec::new(),
8790                    provider_override: None,
8791                    model_override,
8792                    reasoning_override: None,
8793                    workspace: test_workspace(),
8794                    instructions: InstructionBundle::default(),
8795                    developer_context: None,
8796                    task_ledger_required: false,
8797                })
8798                .await
8799                .unwrap();
8800            tokio::time::timeout(std::time::Duration::from_secs(5), async {
8801                loop {
8802                    let envelope = rx.recv().await.unwrap();
8803                    if envelope.turn_id.as_deref() != Some(&turn_id) {
8804                        continue;
8805                    }
8806                    match envelope.event {
8807                        RoderEvent::TurnCompleted(_) => break,
8808                        RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8809                        _ => {}
8810                    }
8811                }
8812            })
8813            .await
8814            .unwrap();
8815        }
8816
8817        let captured = requests.lock().unwrap().clone();
8818        assert_eq!(captured.len(), 2);
8819        assert!(captured[1].transcript.iter().any(|item| {
8820            matches!(
8821                item,
8822                TranscriptItem::UserMessage(message)
8823                    if message.text.starts_with(MODEL_SWITCH_SUMMARY_PREFIX)
8824                        && message.text.contains("previous profile mock/gpt-5.5")
8825                        && message.text.contains("Current profile mock/claude-haiku-4-5-20251001")
8826                        && message.text.contains("Available tools now:")
8827            )
8828        }));
8829
8830        let snapshot = runtime
8831            .thread_store
8832            .as_ref()
8833            .unwrap()
8834            .load_thread(&thread_id)
8835            .await
8836            .unwrap()
8837            .unwrap();
8838        let trace_segments = snapshot
8839            .turns
8840            .iter()
8841            .flat_map(|turn| &turn.items)
8842            .filter(|item| {
8843                matches!(
8844                    item,
8845                    TranscriptItem::ProviderMetadata(value)
8846                        if value.get("kind").and_then(serde_json::Value::as_str)
8847                            == Some(MODEL_PROFILE_TRACE_KIND)
8848                            && value.get("segment").and_then(serde_json::Value::as_str)
8849                                == Some("assistant")
8850                )
8851            })
8852            .count();
8853        assert!(trace_segments >= 2);
8854        let _ = std::fs::remove_dir_all(thread_root);
8855    }
8856
8857    struct CountingTaskTool {
8858        calls: Arc<StdMutex<u32>>,
8859    }
8860
8861    #[async_trait::async_trait]
8862    impl ToolExecutor for CountingTaskTool {
8863        fn spec(&self) -> ToolSpec {
8864            ToolSpec {
8865                name: "task".to_string(),
8866                description: "Dispatch a test subagent.".to_string(),
8867                parameters: serde_json::json!({
8868                    "type": "object",
8869                    "properties": {
8870                        "description": { "type": "string" },
8871                        "prompt": { "type": "string" },
8872                        "parent_deadline_seconds": { "type": "integer" }
8873                    },
8874                    "required": ["description", "prompt"],
8875                    "additionalProperties": false
8876                }),
8877            }
8878        }
8879
8880        async fn execute(
8881            &self,
8882            _ctx: ToolExecutionContext,
8883            call: ToolCall,
8884        ) -> anyhow::Result<ToolResult> {
8885            *self.calls.lock().unwrap() += 1;
8886            Ok(ToolResult {
8887                id: call.id,
8888                name: call.name,
8889                text: "started child".to_string(),
8890                data: serde_json::json!({}),
8891                is_error: false,
8892            })
8893        }
8894    }
8895
8896    #[tokio::test]
8897    async fn runtime_profile_reaches_inference_request_and_turn_metadata() {
8898        let captured = Arc::new(StdMutex::new(None));
8899        let mut builder = ExtensionRegistryBuilder::new();
8900        builder.inference_engine(Arc::new(CapturingEngine {
8901            request: captured.clone(),
8902        }));
8903        let runtime = Arc::new(
8904            Runtime::new(
8905                builder.build().unwrap(),
8906                RuntimeConfig {
8907                    runtime_profile: RuntimeProfile::NonInteractive,
8908                    ..RuntimeConfig::default()
8909                },
8910            )
8911            .unwrap(),
8912        );
8913        let mut rx = runtime.subscribe_events();
8914        let turn_id = runtime
8915            .start_turn(StartTurnRequest {
8916                thread_id: "thread-profile".to_string(),
8917                message: "work unattended".to_string(),
8918                images: Vec::new(),
8919                provider_override: None,
8920                model_override: None,
8921                reasoning_override: None,
8922                workspace: test_workspace(),
8923                instructions: InstructionBundle {
8924                    system: None,
8925                    developer: Some("base developer".to_string()),
8926                    developer_context: None,
8927                },
8928                developer_context: None,
8929                task_ledger_required: false,
8930            })
8931            .await
8932            .unwrap();
8933
8934        let mut observed_profile = None;
8935        tokio::time::timeout(std::time::Duration::from_secs(5), async {
8936            loop {
8937                let envelope = rx.recv().await.unwrap();
8938                if envelope.turn_id.as_deref() != Some(&turn_id) {
8939                    continue;
8940                }
8941                match envelope.event {
8942                    RoderEvent::TurnStarted(event) => {
8943                        observed_profile = Some(event.runtime_profile);
8944                    }
8945                    RoderEvent::TurnCompleted(_) => break,
8946                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8947                    _ => {}
8948                }
8949            }
8950        })
8951        .await
8952        .unwrap();
8953
8954        assert_eq!(observed_profile, Some(RuntimeProfile::NonInteractive));
8955        let request = captured.lock().unwrap().clone().unwrap();
8956        assert_eq!(request.runtime.profile, RuntimeProfile::NonInteractive);
8957        let developer = request.instructions.developer.unwrap();
8958        assert!(developer.contains("base developer"));
8959        assert!(developer.contains("non-interactive profile"));
8960    }
8961
8962    #[tokio::test]
8963    async fn global_policy_mode_changes_do_not_create_runtime_thread_directory() {
8964        let workspace = runtime_test_workspace("global-policy-mode");
8965        let thread_root = workspace.join("threads");
8966        let mut builder = ExtensionRegistryBuilder::new();
8967        builder.inference_engine(Arc::new(FakeInferenceEngine));
8968        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8969            base_path: thread_root.clone(),
8970        }));
8971        let runtime = Runtime::new(
8972            builder.build().unwrap(),
8973            RuntimeConfig {
8974                workspace: Some(workspace.display().to_string()),
8975                ..Default::default()
8976            },
8977        )
8978        .unwrap();
8979
8980        runtime
8981            .set_policy_mode(PolicyMode::AcceptAll, Some("test".to_string()))
8982            .await
8983            .unwrap();
8984
8985        assert!(!thread_root.join("runtime").exists());
8986        let _ = std::fs::remove_dir_all(workspace);
8987    }
8988
8989    #[tokio::test]
8990    async fn task_ledger_enforcement_injects_eval_reminder_before_work() {
8991        let captured = Arc::new(StdMutex::new(None));
8992        let mut builder = ExtensionRegistryBuilder::new();
8993        builder.inference_engine(Arc::new(CapturingEngine {
8994            request: captured.clone(),
8995        }));
8996        builder.tool_contributor(Arc::new(
8997            roder_ext_task_ledger::TaskLedgerToolContributor::default(),
8998        ));
8999        let runtime = Arc::new(
9000            Runtime::new(
9001                builder.build().unwrap(),
9002                RuntimeConfig {
9003                    runtime_profile: RuntimeProfile::Eval,
9004                    policy_mode: PolicyMode::Bypass,
9005                    agent_swarm_mode: false,
9006                    ..RuntimeConfig::default()
9007                },
9008            )
9009            .unwrap(),
9010        );
9011        let mut rx = runtime.subscribe_events();
9012        let turn_id = runtime
9013            .start_turn(StartTurnRequest {
9014                thread_id: "thread-ledger".to_string(),
9015                message: "decomposed work".to_string(),
9016                images: Vec::new(),
9017                provider_override: None,
9018                model_override: None,
9019                reasoning_override: None,
9020                workspace: test_workspace(),
9021                instructions: InstructionBundle::default(),
9022                developer_context: None,
9023                task_ledger_required: true,
9024            })
9025            .await
9026            .unwrap();
9027
9028        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9029            loop {
9030                let envelope = rx.recv().await.unwrap();
9031                if envelope.turn_id.as_deref() == Some(&turn_id)
9032                    && matches!(envelope.event, RoderEvent::TurnCompleted(_))
9033                {
9034                    break;
9035                }
9036            }
9037        })
9038        .await
9039        .unwrap();
9040
9041        let request = captured.lock().unwrap().clone().unwrap();
9042        let developer = request.instructions.developer.unwrap();
9043        assert!(developer.contains("Task Ledger Required"));
9044        assert!(developer.contains("task_ledger.update"));
9045        let tool_names: Vec<_> = request
9046            .tools
9047            .iter()
9048            .map(|tool| tool.name.as_str())
9049            .collect();
9050        assert!(
9051            tool_names.contains(&TASK_LEDGER_TOOL_NAME),
9052            "tool names: {tool_names:?}"
9053        );
9054        assert_eq!(
9055            request.tool_choice,
9056            ToolChoice::Specific(TASK_LEDGER_TOOL_NAME.to_string())
9057        );
9058        assert_eq!(request.tools.len(), 1);
9059        assert_eq!(request.tools[0].name, TASK_LEDGER_TOOL_NAME);
9060    }
9061
9062    #[tokio::test]
9063    async fn eval_task_ledger_blocks_final_answer_until_open_items_are_completed() {
9064        let requests = Arc::new(StdMutex::new(Vec::new()));
9065        let mut builder = ExtensionRegistryBuilder::new();
9066        builder.inference_engine(Arc::new(TaskLedgerCompletionGateEngine {
9067            calls: StdMutex::new(0),
9068            requests: requests.clone(),
9069        }));
9070        builder.tool_contributor(Arc::new(
9071            roder_ext_task_ledger::TaskLedgerToolContributor::default(),
9072        ));
9073        let runtime = Arc::new(
9074            Runtime::new(
9075                builder.build().unwrap(),
9076                RuntimeConfig {
9077                    runtime_profile: RuntimeProfile::Eval,
9078                    policy_mode: PolicyMode::Bypass,
9079                    agent_swarm_mode: false,
9080                    ..RuntimeConfig::default()
9081                },
9082            )
9083            .unwrap(),
9084        );
9085        let mut rx = runtime.subscribe_events();
9086        let turn_id = runtime
9087            .start_turn(StartTurnRequest {
9088                thread_id: "thread-ledger-completion".to_string(),
9089                message: "write the answer file".to_string(),
9090                images: Vec::new(),
9091                provider_override: None,
9092                model_override: None,
9093                reasoning_override: None,
9094                workspace: test_workspace(),
9095                instructions: InstructionBundle::default(),
9096                developer_context: None,
9097                task_ledger_required: true,
9098            })
9099            .await
9100            .unwrap();
9101
9102        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9103            loop {
9104                let envelope = rx.recv().await.unwrap();
9105                if envelope.turn_id.as_deref() != Some(&turn_id) {
9106                    continue;
9107                }
9108                match envelope.event {
9109                    RoderEvent::TurnCompleted(_) => break,
9110                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9111                    _ => {}
9112                }
9113            }
9114        })
9115        .await
9116        .unwrap();
9117
9118        let requests = requests.lock().unwrap().clone();
9119        assert_eq!(requests.len(), 4);
9120        assert!(requests[2].transcript.iter().any(|item| {
9121            matches!(
9122                item,
9123                TranscriptItem::UserMessage(message)
9124                    if message.text.contains("Task Ledger Completion Required")
9125                        && message.text.contains("Write /app/result.txt")
9126            )
9127        }));
9128        assert!(requests[3].transcript.iter().any(|item| {
9129            matches!(
9130                item,
9131                TranscriptItem::ToolResult(result)
9132                    if result.name.as_deref() == Some(TASK_LEDGER_TOOL_NAME)
9133                        && result.result.contains("Task ledger: 2/2 completed")
9134            )
9135        }));
9136    }
9137
9138    #[tokio::test]
9139    async fn eval_task_ledger_checkpoint_requests_scoreable_file_before_final_reserve() {
9140        let requests = Arc::new(StdMutex::new(Vec::new()));
9141        let mut builder = ExtensionRegistryBuilder::new();
9142        builder.inference_engine(Arc::new(TaskLedgerCompletionGateEngine {
9143            calls: StdMutex::new(0),
9144            requests: requests.clone(),
9145        }));
9146        builder.tool_contributor(Arc::new(
9147            roder_ext_task_ledger::TaskLedgerToolContributor::default(),
9148        ));
9149        let runtime = Arc::new(
9150            Runtime::new(
9151                builder.build().unwrap(),
9152                RuntimeConfig {
9153                    runtime_profile: RuntimeProfile::Eval,
9154                    policy_mode: PolicyMode::Bypass,
9155                    agent_swarm_mode: false,
9156                    turn_deadline_seconds: Some(120),
9157                    ..RuntimeConfig::default()
9158                },
9159            )
9160            .unwrap(),
9161        );
9162        let mut rx = runtime.subscribe_events();
9163        let turn_id = runtime
9164            .start_turn(StartTurnRequest {
9165                thread_id: "thread-ledger-checkpoint".to_string(),
9166                message: "write the answer file".to_string(),
9167                images: Vec::new(),
9168                provider_override: None,
9169                model_override: None,
9170                reasoning_override: None,
9171                workspace: test_workspace(),
9172                instructions: InstructionBundle::default(),
9173                developer_context: None,
9174                task_ledger_required: true,
9175            })
9176            .await
9177            .unwrap();
9178
9179        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9180            loop {
9181                let envelope = rx.recv().await.unwrap();
9182                if envelope.turn_id.as_deref() != Some(&turn_id) {
9183                    continue;
9184                }
9185                match envelope.event {
9186                    RoderEvent::TurnCompleted(_) => break,
9187                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9188                    _ => {}
9189                }
9190            }
9191        })
9192        .await
9193        .unwrap();
9194
9195        let requests = requests.lock().unwrap().clone();
9196        assert!(requests.len() >= 2);
9197        assert!(requests[1].transcript.iter().any(|item| {
9198            matches!(
9199                item,
9200                TranscriptItem::UserMessage(message)
9201                    if message.text.contains("Scoreable Output Checkpoint")
9202                        && message.text.contains("ensure the required output file(s) exist")
9203                        && message.text.contains("Write /app/result.txt")
9204            )
9205        }));
9206    }
9207
9208    #[test]
9209    fn deadline_task_ledger_prompt_preserves_scoreable_work_instruction() {
9210        let prompt = task_ledger_deadline_completion_prompt(
9211            12,
9212            30,
9213            "Task Ledger Completion Required: write /app/result.txt, then call task_ledger.update",
9214        );
9215
9216        assert!(prompt.contains("12 seconds remain"));
9217        assert!(prompt.contains("create or update the required scoreable output files"));
9218        assert!(prompt.contains("write /app/result.txt"));
9219        assert!(prompt.contains(TASK_LEDGER_TOOL_NAME));
9220    }
9221
9222    #[test]
9223    fn scoreable_checkpoint_prompt_preserves_provisional_file_instruction() {
9224        let prompt = task_ledger_scoreable_checkpoint_prompt(
9225            120,
9226            "Task Ledger Completion Required: write /app/result.txt, then call task_ledger.update",
9227        );
9228
9229        assert!(prompt.contains("120 seconds remain"));
9230        assert!(prompt.contains("best evidence-backed answer"));
9231        assert!(prompt.contains("even if provisional"));
9232        assert!(prompt.contains("preserve that candidate"));
9233        assert!(prompt.contains("partial-coverage"));
9234        assert!(prompt.contains("write /app/result.txt"));
9235        assert!(prompt.contains(TASK_LEDGER_TOOL_NAME));
9236    }
9237
9238    #[test]
9239    fn open_task_ledger_moves_inference_timeout_to_scoreable_checkpoint() {
9240        let deadline = Some(OffsetDateTime::now_utc() + Duration::seconds(870));
9241        let transcript = vec![TranscriptItem::ToolResult(ToolResultRecord {
9242            id: "ledger-open".to_string(),
9243            name: Some(TASK_LEDGER_TOOL_NAME.to_string()),
9244            result: "Task ledger: 0/1 completed\n- pending: Write /app/result.txt [write]"
9245                .to_string(),
9246            display_payload: None,
9247            is_error: false,
9248        })];
9249
9250        let (_, action) = inference_timeout_deadline(
9251            deadline,
9252            RuntimeProfile::Eval,
9253            true,
9254            30,
9255            false,
9256            0,
9257            &transcript,
9258        )
9259        .unwrap();
9260
9261        assert_eq!(action, InferenceTimeoutAction::ScoreableCheckpoint);
9262    }
9263
9264    #[tokio::test]
9265    async fn verification_gate_forces_eval_code_changes_through_review() {
9266        let mut builder = ExtensionRegistryBuilder::new();
9267        builder.inference_engine(Arc::new(VerificationGateEngine {
9268            calls: StdMutex::new(0),
9269        }));
9270        builder.tool_contributor(Arc::new(WriteFileContributor));
9271        builder.tool_contributor(Arc::new(
9272            roder_ext_verification::VerificationToolContributor,
9273        ));
9274        let runtime = Arc::new(
9275            Runtime::new(
9276                builder.build().unwrap(),
9277                RuntimeConfig {
9278                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9279                    default_model: "mock".to_string(),
9280                    runtime_profile: RuntimeProfile::Eval,
9281                    policy_mode: PolicyMode::Bypass,
9282                    agent_swarm_mode: false,
9283                    ..RuntimeConfig::default()
9284                },
9285            )
9286            .unwrap(),
9287        );
9288        let mut rx = runtime.subscribe_events();
9289        let turn_id = runtime
9290            .start_turn(StartTurnRequest {
9291                thread_id: "thread-verification".to_string(),
9292                message: "write code".to_string(),
9293                images: Vec::new(),
9294                provider_override: None,
9295                model_override: None,
9296                reasoning_override: None,
9297                workspace: test_workspace(),
9298                instructions: InstructionBundle::default(),
9299                developer_context: None,
9300                task_ledger_required: false,
9301            })
9302            .await
9303            .unwrap();
9304
9305        let mut saw_required = false;
9306        let mut saw_completed = false;
9307        let mut final_text = String::new();
9308        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9309            loop {
9310                let envelope = rx.recv().await.unwrap();
9311                if envelope.turn_id.as_deref() != Some(&turn_id) {
9312                    continue;
9313                }
9314                match envelope.event {
9315                    RoderEvent::VerificationRequired(event) => {
9316                        saw_required = true;
9317                        assert_eq!(event.changed_files, vec!["src/lib.rs"]);
9318                    }
9319                    RoderEvent::VerificationCompleted(event) => {
9320                        saw_completed = true;
9321                        assert!(event.passed);
9322                    }
9323                    RoderEvent::InferenceEventReceived(event) => {
9324                        if let InferenceEvent::MessageDelta(delta) = event.event {
9325                            final_text.push_str(&delta.text);
9326                        }
9327                    }
9328                    RoderEvent::TurnCompleted(_) => break,
9329                    _ => {}
9330                }
9331            }
9332        })
9333        .await
9334        .unwrap();
9335
9336        assert!(saw_required);
9337        assert!(saw_completed);
9338        assert!(final_text.contains("verified final"));
9339    }
9340
9341    #[tokio::test]
9342    async fn speed_policy_changes_reasoning_across_eval_model_calls_without_model_switch() {
9343        let requests = Arc::new(StdMutex::new(Vec::new()));
9344        let mut builder = ExtensionRegistryBuilder::new();
9345        builder.inference_engine(Arc::new(SpeedPolicyEngine {
9346            calls: StdMutex::new(0),
9347            requests: requests.clone(),
9348        }));
9349        builder.tool_contributor(Arc::new(WriteFileContributor));
9350        builder.tool_contributor(Arc::new(
9351            roder_ext_verification::VerificationToolContributor,
9352        ));
9353        let runtime = Arc::new(
9354            Runtime::new(
9355                builder.build().unwrap(),
9356                RuntimeConfig {
9357                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9358                    default_model: "gpt-5.5".to_string(),
9359                    runtime_profile: RuntimeProfile::Eval,
9360                    policy_mode: PolicyMode::Bypass,
9361                    agent_swarm_mode: false,
9362                    ..RuntimeConfig::default()
9363                },
9364            )
9365            .unwrap(),
9366        );
9367        let mut rx = runtime.subscribe_events();
9368        let turn_id = runtime
9369            .start_turn(StartTurnRequest {
9370                thread_id: "thread-speed-policy".to_string(),
9371                message: "write code".to_string(),
9372                images: Vec::new(),
9373                provider_override: None,
9374                model_override: None,
9375                reasoning_override: None,
9376                workspace: test_workspace(),
9377                instructions: InstructionBundle::default(),
9378                developer_context: None,
9379                task_ledger_required: false,
9380            })
9381            .await
9382            .unwrap();
9383
9384        let mut saw_speed_policy_event = false;
9385        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9386            loop {
9387                let envelope = rx.recv().await.unwrap();
9388                if envelope.turn_id.as_deref() != Some(&turn_id) {
9389                    continue;
9390                }
9391                match envelope.event {
9392                    RoderEvent::InferenceStarted(event) => {
9393                        if event.speed_policy.is_some() {
9394                            saw_speed_policy_event = true;
9395                        }
9396                    }
9397                    RoderEvent::TurnCompleted(_) => break,
9398                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9399                    _ => {}
9400                }
9401            }
9402        })
9403        .await
9404        .unwrap();
9405
9406        let requests = requests.lock().unwrap().clone();
9407        assert!(saw_speed_policy_event);
9408        assert!(requests.len() >= 4);
9409        assert!(requests.iter().all(|request| {
9410            request.model.provider == roder_api::catalog::PROVIDER_MOCK
9411                && request.model.model == "gpt-5.5"
9412        }));
9413        assert_eq!(
9414            requests[0].runtime.speed_policy.as_ref().map(|d| d.phase),
9415            Some(roder_api::inference::SpeedPolicyPhase::Orientation)
9416        );
9417        assert_eq!(requests[0].reasoning.level.as_deref(), Some(REASONING_HIGH));
9418        assert_eq!(
9419            requests[1].runtime.speed_policy.as_ref().map(|d| d.phase),
9420            Some(roder_api::inference::SpeedPolicyPhase::Execution)
9421        );
9422        assert_eq!(requests[1].reasoning.level.as_deref(), Some(REASONING_LOW));
9423        assert_eq!(
9424            requests[2].runtime.speed_policy.as_ref().map(|d| d.phase),
9425            Some(roder_api::inference::SpeedPolicyPhase::Verification)
9426        );
9427        assert_eq!(requests[2].reasoning.level.as_deref(), Some(REASONING_HIGH));
9428        assert_eq!(
9429            requests[2]
9430                .metadata
9431                .pointer("/speedPolicy/phase")
9432                .and_then(serde_json::Value::as_str),
9433            Some("verification")
9434        );
9435    }
9436
9437    #[tokio::test]
9438    async fn deadline_turn_timeout_emits_partial_result_and_clears_active_turn() {
9439        let mut builder = ExtensionRegistryBuilder::new();
9440        builder.inference_engine(Arc::new(DeadlineEngine));
9441        let runtime = Arc::new(
9442            Runtime::new(
9443                builder.build().unwrap(),
9444                RuntimeConfig {
9445                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9446                    default_model: "mock".to_string(),
9447                    runtime_profile: RuntimeProfile::Eval,
9448                    turn_deadline_seconds: Some(1),
9449                    ..RuntimeConfig::default()
9450                },
9451            )
9452            .unwrap(),
9453        );
9454        let mut rx = runtime.subscribe_events();
9455        let turn_id = runtime
9456            .start_turn(StartTurnRequest {
9457                thread_id: "thread-deadline".to_string(),
9458                message: "slow work".to_string(),
9459                images: Vec::new(),
9460                provider_override: None,
9461                model_override: None,
9462                reasoning_override: None,
9463                workspace: test_workspace(),
9464                instructions: InstructionBundle::default(),
9465                developer_context: None,
9466                task_ledger_required: false,
9467            })
9468            .await
9469            .unwrap();
9470
9471        let mut saw_partial = false;
9472        let mut saw_deadline = false;
9473        let mut failed_kind = None;
9474        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9475            loop {
9476                let envelope = rx.recv().await.unwrap();
9477                if envelope.turn_id.as_deref() != Some(&turn_id) {
9478                    continue;
9479                }
9480                match envelope.event {
9481                    RoderEvent::TurnPartialResult(event) => {
9482                        saw_partial = event.summary.contains("partial turn state");
9483                    }
9484                    RoderEvent::TurnDeadlineExceeded(event) => {
9485                        saw_deadline = event.partial_result.contains("transcript items");
9486                    }
9487                    RoderEvent::TurnFailed(event) => {
9488                        failed_kind = event.error_kind;
9489                        break;
9490                    }
9491                    _ => {}
9492                }
9493            }
9494        })
9495        .await
9496        .unwrap();
9497
9498        for _ in 0..20 {
9499            if !runtime.active_turns.read().await.contains_key(&turn_id) {
9500                break;
9501            }
9502            tokio::time::sleep(std::time::Duration::from_millis(10)).await;
9503        }
9504        assert!(saw_partial);
9505        assert!(saw_deadline);
9506        assert_eq!(failed_kind.as_deref(), Some("deadline_timeout"));
9507        assert!(!runtime.active_turns.read().await.contains_key(&turn_id));
9508    }
9509
9510    #[tokio::test]
9511    async fn deadline_skips_subagent_task_when_remaining_budget_is_too_low() {
9512        let calls = Arc::new(StdMutex::new(0));
9513        let mut builder = ExtensionRegistryBuilder::new();
9514        builder.inference_engine(Arc::new(CapturingEngine {
9515            request: Arc::new(StdMutex::new(None)),
9516        }));
9517        let task_tool = Arc::new(CountingTaskTool {
9518            calls: calls.clone(),
9519        });
9520        builder.tool_contributor(Arc::new(TestToolContributor { tool: task_tool }));
9521        let runtime = Arc::new(
9522            Runtime::new(
9523                builder.build().unwrap(),
9524                RuntimeConfig {
9525                    policy_mode: PolicyMode::Bypass,
9526                    agent_swarm_mode: false,
9527                    ..RuntimeConfig::default()
9528                },
9529            )
9530            .unwrap(),
9531        );
9532
9533        let result = runtime
9534            .route_tool_call(
9535                &"thread-deadline-task".to_string(),
9536                &"turn-deadline-task".to_string(),
9537                ToolCallCompleted {
9538                    id: "task-1".to_string(),
9539                    name: "task".to_string(),
9540                    arguments: serde_json::json!({
9541                        "description": "inspect",
9542                        "prompt": "read"
9543                    })
9544                    .to_string(),
9545                },
9546                None,
9547                Some(OffsetDateTime::now_utc() + Duration::seconds(1)),
9548            )
9549            .await
9550            .unwrap();
9551
9552        assert!(result.is_error);
9553        assert!(result.result.contains("deadline policy skipped"));
9554        assert_eq!(*calls.lock().unwrap(), 0);
9555    }
9556
9557    struct TestToolContributor {
9558        tool: Arc<dyn ToolExecutor>,
9559    }
9560
9561    impl ToolContributor for TestToolContributor {
9562        fn id(&self) -> String {
9563            "test-tool".to_string()
9564        }
9565
9566        fn contribute(&self, registry: &mut ToolRegistry) -> anyhow::Result<()> {
9567            registry.register(self.tool.clone())
9568        }
9569    }
9570
9571    #[tokio::test]
9572    async fn agent_swarm_mode_override_is_per_thread() {
9573        let runtime = Runtime::fake().unwrap();
9574        let trigger = roder_api::subagents::AgentSwarmModeTrigger::Manual;
9575
9576        // Off everywhere by default.
9577        assert!(
9578            !runtime
9579                .effective_agent_swarm_mode_for_thread("thread-a")
9580                .await
9581        );
9582        assert!(
9583            !runtime
9584                .effective_agent_swarm_mode_for_thread("thread-b")
9585                .await
9586        );
9587
9588        // Enabling on thread-a does not leak into thread-b.
9589        assert!(
9590            runtime
9591                .set_agent_swarm_mode_for_thread("thread-a", true, trigger)
9592                .await
9593        );
9594        assert!(
9595            runtime
9596                .effective_agent_swarm_mode_for_thread("thread-a")
9597                .await
9598        );
9599        assert!(
9600            !runtime
9601                .effective_agent_swarm_mode_for_thread("thread-b")
9602                .await
9603        );
9604
9605        // A per-thread `off` override wins over a runtime-global `on` default.
9606        runtime.set_agent_swarm_mode(true, trigger).await.unwrap();
9607        runtime
9608            .set_agent_swarm_mode_for_thread("thread-b", false, trigger)
9609            .await;
9610        assert!(
9611            !runtime
9612                .effective_agent_swarm_mode_for_thread("thread-b")
9613                .await,
9614            "explicit per-thread off overrides the global on default"
9615        );
9616        // A thread with no override still follows the global default.
9617        assert!(
9618            runtime
9619                .effective_agent_swarm_mode_for_thread("thread-c")
9620                .await,
9621            "threads without an override follow the runtime-global default"
9622        );
9623    }
9624
9625    #[tokio::test]
9626    async fn set_agent_swarm_mode_for_thread_emits_event_with_real_thread_id() {
9627        let runtime = Runtime::fake().unwrap();
9628        let mut events = runtime.subscribe_events();
9629        runtime
9630            .set_agent_swarm_mode_for_thread(
9631                "thread-xyz",
9632                true,
9633                roder_api::subagents::AgentSwarmModeTrigger::Task,
9634            )
9635            .await;
9636        let mut saw = false;
9637        for _ in 0..8 {
9638            let envelope = tokio::time::timeout(std::time::Duration::from_secs(2), events.recv())
9639                .await
9640                .unwrap()
9641                .unwrap();
9642            if let RoderEvent::AgentSwarmModeChanged(event) = envelope.event {
9643                assert_eq!(event.thread_id, "thread-xyz");
9644                assert!(event.enabled);
9645                assert_eq!(
9646                    event.trigger,
9647                    roder_api::subagents::AgentSwarmModeTrigger::Task
9648                );
9649                saw = true;
9650                break;
9651            }
9652        }
9653        assert!(
9654            saw,
9655            "expected an AgentSwarmModeChanged event for the thread"
9656        );
9657    }
9658}