Skip to main content

roder_core/
runtime.rs

1use std::collections::HashMap;
2use std::path::PathBuf;
3use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
4use std::sync::{Arc, Weak};
5
6use anyhow::Context;
7use futures::StreamExt;
8use futures::future::{AbortHandle, Abortable, BoxFuture, try_join_all};
9use roder_api::catalog::{
10    EDIT_TOOL_EDIT, EDIT_TOOL_PATCH, PROVIDER_GEMINI, REASONING_NONE, REASONING_ULTRA,
11    built_in_model_profile, built_in_model_profile_for_provider, lookup_model,
12    model_supports_reasoning_effort,
13};
14use roder_api::context::PolicyGate;
15use roder_api::events::*;
16use roder_api::extension::ExtensionRegistry;
17use roder_api::inference::{
18    AgentInferenceRequest, HostedWebSearchConfig, HostedWebSearchMode, InferenceEngine,
19    InferenceEvent, InferenceTurnContext, InstructionBundle, ModelHarnessProfile,
20    ModelSchemaPolicy, ModelSelection, OutputConfig, ProviderTurnCleanup, ReasoningConfig,
21    RuntimeHints, RuntimeProfile, TokenUsage, ToolCallCompleted, ToolSearchConfig,
22    ToolSearchConfigOverlay, finish_reason_from_stop_reason,
23};
24use roder_api::inference_routing::{InferenceRoutingOutcome, ModelSelectionMode};
25use roder_api::lifecycle::{
26    LifecycleMetricsSnapshot, TurnCleanupOwnership, TurnCleanupState, TurnLifecycleReason,
27    TurnLifecycleRecord, TurnLifecycleSnapshot, TurnLifecycleState, turn_lifecycle_snapshot,
28};
29use roder_api::policy_mode::{PolicyDecision, PolicyMode};
30use roder_api::reliability::{
31    ReliabilityContext, ReliabilityDetails, ReliabilityErrorClass, ReliabilityLimitDecision,
32    ReliabilityLimitRecorded, ReliabilityRequestPolicy, ReliabilityRetryDecision,
33    ReliabilityRetryRecorded, provider_retry_delay_ms,
34};
35use roder_api::remote_runner::{
36    RemoteRunnerProvider, RemoteRunnerSession, RemoteWorkspace, RunnerDestination,
37    RunnerSessionState, ThreadRunnerBinding,
38};
39use roder_api::subagents::SubagentDefinition;
40use roder_api::teams::{
41    TeamId, TeamMailboxMessage, TeamMailboxMessageKind, TeamMemberDescriptor, TeamMemberRole,
42    TeamMemberStatus,
43};
44use roder_api::thread::{
45    ThreadItemEvent, ThreadItemEventKind, ThreadMetadata, ThreadSnapshot, ThreadStore,
46    ThreadUsageMetadata, is_synthetic_event_thread_id, validate_thread_workspace,
47};
48use roder_api::tools::{ToolCall, ToolChoice, ToolExecutionContext, ToolRegistry, ToolResult};
49use roder_api::transcript::{
50    AssistantMessage, ErrorRecord, InputImage, ReasoningSummary, ToolCallRecord, ToolResultRecord,
51    TranscriptItem, UserMessage,
52};
53use roder_sandbox::ScopedFilesystem;
54use roder_sandbox::process::LocalProcessRunner;
55use roder_skills::{SkillRegistry, SkillRegistryOptions};
56use time::{Duration, OffsetDateTime};
57use tokio::sync::{Mutex, Notify, RwLock, oneshot};
58
59mod codex_v2;
60
61use crate::artifacts::{
62    ContextArtifactStore as FilesystemContextArtifactStore, default_context_artifact_dir,
63};
64use crate::bus::EventBus;
65use crate::dynamic_workflows::{
66    DynamicWorkflowEffortProfile, RuntimeDynamicWorkflowConfig, WorkflowTriggerDecision,
67    classify_workflow_trigger, ultracode_reasoning_level_for_model,
68};
69use crate::fake_provider::FakeInferenceEngine;
70use crate::goals::RuntimeGoalController;
71use crate::inference_routing::{
72    InferenceRoutingRequest, RuntimeInferenceRouterConfig, collect_inference_routing_candidates,
73    route_inference_selection, transcript_failure_count_since,
74};
75use crate::instructions::{
76    apply_agent_swarm_mode, apply_codex_multi_agent_mode, apply_model_instruction_overlay,
77    apply_parallel_web_tools, apply_plan_mode, apply_runtime_profile, apply_task_ledger_required,
78    apply_thread_developer_instructions, apply_turn_developer_context,
79};
80use crate::policy_gate::DefaultPolicyGate;
81use crate::reliability::{
82    ReliabilityLimitHit, RuntimeReliabilityConfig, TurnReliabilityState,
83    provider_stream_retry_cause,
84};
85pub use crate::speed_policy::RuntimeSpeedPolicyConfig;
86use crate::speed_policy::{SpeedPolicyState, reasoning_from_decision};
87use crate::subagent_traces::{RuntimeAgentSwarmProgressSink, RuntimeSubagentTraceSink};
88use crate::teams::{TeamManager, TeamMemberStartRequest, TeamStartRequest, TeamState};
89use crate::thread_item_cache::{ThreadItemCache, ThreadItemCacheEntry};
90use crate::verification_gate::VerificationGateState;
91
92const MAX_TOOL_ROUNDS_PER_TURN: usize = 1024;
93/// Injected when `continue_on_failure_limit` turns a consecutive-tool-failure
94/// limit into a continuation, or as the empty-tool-call persistence nudge, so
95/// the model keeps working instead of ending the turn early.
96const RELIABILITY_CONTINUATION_PROMPT: &str = "Verify the task is fully complete. If any part remains unfinished or unverified, keep working using the available tools — try an alternative approach if one is failing. Only stop once the solution is complete and verified; if it is already complete, restate your final answer.";
97/// Capacity of the runtime event broadcast ring buffer. The TUI drains this on
98/// its render loop, which during an active turn only wakes at the ~6 FPS status
99/// animation cadence (backend events do not wake the input poll), so up to
100/// ~166ms of events buffer between drains. A reasoning-high provider that runs
101/// its whole tool loop inside one turn (e.g. claude-code) streams a firehose of
102/// `InferenceEventReceived` deltas plus per-step tool/thinking events; the old
103/// 1024-slot buffer overflowed in those windows and silently dropped tool and
104/// thinking rows from the live view. Sized for generous headroom across bursts
105/// and brief render stalls.
106const EVENT_BUS_CAPACITY: usize = 16_384;
107pub(crate) const FINAL_ANSWER_PHASE: &str = "final_answer";
108pub(crate) const TASK_LEDGER_TOOL_NAME: &str = "task_ledger.update";
109const TASK_LEDGER_COMPLETION_REMINDER_LIMIT: u8 = 2;
110const TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS: u64 = 180;
111const TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT: u8 = 1;
112pub(crate) const MIN_CHILD_DEADLINE_SECONDS: u64 = 2;
113const MODEL_PROFILE_TRACE_KIND: &str = "model_profile_segment";
114const MODEL_SWITCH_SUMMARY_PREFIX: &str = "Model switch summary:";
115const PROVIDER_CLEANUP_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5);
116
117#[derive(Clone, Copy, Debug, Eq, PartialEq)]
118enum InferenceTimeoutAction {
119    ScoreableCheckpoint,
120    Finalization,
121}
122
123#[derive(Debug, Clone)]
124pub struct RuntimeConfig {
125    pub default_provider: String,
126    pub default_model: String,
127    pub reasoning: Option<String>,
128    pub auto_compact_token_limit: Option<u32>,
129    pub file_backed_dynamic_context: bool,
130    pub hosted_web_search: HostedWebSearchConfig,
131    pub tool_search: ToolSearchConfig,
132    pub provider_tool_search: HashMap<String, ToolSearchConfigOverlay>,
133    pub model_tool_search: HashMap<String, ToolSearchConfigOverlay>,
134    pub model_edit_tools: HashMap<String, String>,
135    pub model_parallel_tool_calls: HashMap<String, bool>,
136    pub model_profiles: HashMap<String, ModelHarnessProfile>,
137    pub tool_allowlist: Vec<String>,
138    /// Seconds a host-executed external tool call may stay unresolved before it fails with a timeout error.
139    pub external_tool_timeout_seconds: u64,
140    pub command_shell: String,
141    pub workspace: Option<String>,
142    pub policy_mode: PolicyMode,
143    /// Whether agent-swarm mode is active (roadmap 104). When on, the runtime
144    /// injects the swarm reminder into each turn's developer instructions so any
145    /// client benefits, not just the TUI.
146    pub agent_swarm_mode: bool,
147    pub runtime_profile: RuntimeProfile,
148    pub inference_router: RuntimeInferenceRouterConfig,
149    pub speed_policy: RuntimeSpeedPolicyConfig,
150    pub dynamic_workflows: RuntimeDynamicWorkflowConfig,
151    pub reliability: RuntimeReliabilityConfig,
152    /// Positive wall-clock limit for a turn in any runtime profile. `None` or zero is unbounded.
153    pub turn_deadline_seconds: Option<u64>,
154    pub remote_runner_destination: Option<RunnerDestination>,
155    pub team_data_dir: Option<PathBuf>,
156    pub roadmap_data_dir: Option<PathBuf>,
157    pub media_generation: crate::media_generation::RuntimeMediaGenerationConfig,
158    /// `[review]` settings: the review sub-turn and its publishers.
159    pub review: crate::review::RuntimeReviewConfig,
160}
161
162impl Default for RuntimeConfig {
163    fn default() -> Self {
164        Self {
165            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
166            default_model: "mock".to_string(),
167            reasoning: None,
168            auto_compact_token_limit: None,
169            file_backed_dynamic_context: true,
170            hosted_web_search: HostedWebSearchConfig::cached(),
171            tool_search: ToolSearchConfig::default(),
172            provider_tool_search: HashMap::new(),
173            model_tool_search: HashMap::new(),
174            model_edit_tools: HashMap::new(),
175            model_parallel_tool_calls: HashMap::new(),
176            model_profiles: HashMap::new(),
177            tool_allowlist: Vec::new(),
178            external_tool_timeout_seconds: DEFAULT_EXTERNAL_TOOL_TIMEOUT_SECONDS,
179            command_shell: roder_api::command_shell::default_command_shell(),
180            workspace: None,
181            policy_mode: PolicyMode::Default,
182            agent_swarm_mode: false,
183            runtime_profile: RuntimeProfile::Interactive,
184            inference_router: RuntimeInferenceRouterConfig::default(),
185            speed_policy: RuntimeSpeedPolicyConfig::default(),
186            dynamic_workflows: RuntimeDynamicWorkflowConfig::default(),
187            reliability: RuntimeReliabilityConfig::default(),
188            turn_deadline_seconds: None,
189            remote_runner_destination: None,
190            team_data_dir: None,
191            roadmap_data_dir: None,
192            media_generation: crate::media_generation::RuntimeMediaGenerationConfig::default(),
193            review: crate::review::RuntimeReviewConfig::default(),
194        }
195    }
196}
197
198#[derive(Debug, Clone)]
199pub struct StartTurnRequest {
200    pub thread_id: ThreadId,
201    pub message: String,
202    pub images: Vec<InputImage>,
203    pub provider_override: Option<String>,
204    pub model_override: Option<String>,
205    pub reasoning_override: Option<String>,
206    pub workspace: String,
207    pub instructions: InstructionBundle,
208    /**
209     * Per-turn developer-authority context for this turn's InstructionBundle.
210     * Applies to every inference round of the turn, is never written to
211     * thread state, and does not carry over to later turns.
212     */
213    pub developer_context: Option<String>,
214    pub task_ledger_required: bool,
215}
216
217#[derive(Debug, Clone)]
218pub struct CreateThreadRequest {
219    pub title: Option<String>,
220    pub workspace: String,
221    pub workspace_id: Option<String>,
222    pub root_id: Option<String>,
223    pub provider: Option<String>,
224    pub model: Option<String>,
225    pub selection_mode: Option<ModelSelectionMode>,
226    /// Per-thread tool filter applied on top of the runtime allowlist. Empty = no filtering.
227    pub tool_allowlist: Vec<String>,
228    /// Host-supplied instructions added to the developer slot of every turn's inference request.
229    pub developer_instructions: Option<String>,
230    /// Host-executed tool specs advertised to the model on every turn of this thread.
231    pub external_tools: Vec<roder_api::tools::ToolSpec>,
232    /// Explicit remote-runner binding for the thread's native coding tools.
233    pub runner: Option<ThreadRunnerSelection>,
234}
235
236/**
237 * Thread-level remote-runner selection. The destination config is persisted
238 * with the thread, so secrets must reach the provider through its
239 * environment, not this config.
240 */
241#[derive(Debug, Clone)]
242pub struct ThreadRunnerSelection {
243    pub provider_id: String,
244    pub config: serde_json::Value,
245    /// Absolute path on the runner used as the thread's coding-tool workspace root.
246    pub workspace: String,
247    /**
248     * Extra absolute runner paths file reads may resolve under, beyond
249     * `workspace`. Writes and the working directory stay confined to
250     * `workspace`.
251     */
252    pub read_roots: Vec<String>,
253}
254
255#[derive(Debug, Clone, PartialEq, Eq)]
256pub struct PendingPlanExit {
257    pub thread_id: ThreadId,
258    pub turn_id: TurnId,
259    pub request_id: String,
260    pub target_mode: PolicyMode,
261    pub plan_summary: Option<String>,
262    pub next_steps: Vec<String>,
263    pub requested_at: OffsetDateTime,
264    pub expires_at: Option<OffsetDateTime>,
265}
266
267pub(crate) struct PendingToolApproval {
268    pub(crate) thread_id: ThreadId,
269    pub(crate) turn_id: TurnId,
270    pub(crate) tool_id: String,
271    pub(crate) tool_name: String,
272    pub(crate) call: roder_api::tools::ToolCall,
273    pub(crate) tx: oneshot::Sender<bool>,
274}
275
276pub(crate) struct PendingUserInput {
277    pub(crate) thread_id: ThreadId,
278    pub(crate) turn_id: TurnId,
279    pub(crate) tx: oneshot::Sender<serde_json::Value>,
280}
281
282/// Host answer to an external tool call delivered via `tools/resolve`.
283#[derive(Debug, Clone, PartialEq, Eq)]
284pub struct ExternalToolResolution {
285    pub output: String,
286    pub is_error: bool,
287}
288
289pub(crate) struct PendingExternalToolCall {
290    pub(crate) thread_id: ThreadId,
291    pub(crate) turn_id: TurnId,
292    pub(crate) tool_id: String,
293    pub(crate) tool_name: String,
294    pub(crate) tx: oneshot::Sender<ExternalToolResolution>,
295}
296
297#[derive(Clone)]
298struct ActiveTurnHandle {
299    thread_id: ThreadId,
300    abort: AbortHandle,
301    steers: Arc<Mutex<Vec<QueuedTurnSteer>>>,
302    drain: Arc<TurnDrainHandle>,
303}
304
305/// Tracks a task after it has been interrupted and removed from the interactive
306/// active-turn map. This lets users start a follow-up turn immediately while a
307/// bounded runtime shutdown can still await the original task's cleanup.
308struct TurnDrainHandle {
309    thread_id: ThreadId,
310    interrupt_requested: AtomicBool,
311    interrupt_reason: Mutex<Option<TurnLifecycleReason>>,
312    completed: AtomicBool,
313    completed_notify: Notify,
314}
315
316/// Result of a bounded runtime drain. `Clean` proves the runtime's own turn
317/// tasks have reached their cleanup paths; provider-specific child-process
318/// reaping remains an explicit provider capability.
319#[derive(Debug, Clone, PartialEq, Eq)]
320pub enum RuntimeDrainOutcome {
321    Clean {
322        interrupted_turn_ids: Vec<TurnId>,
323    },
324    DeadlineExceeded {
325        interrupted_turn_ids: Vec<TurnId>,
326        remaining_turn_ids: Vec<TurnId>,
327    },
328    /// At least one lifecycle transition initiated by this drain could not be
329    /// durably appended. The runtime still made its best cleanup attempt, but
330    /// callers must not claim a persisted terminal state as proof of recovery.
331    PersistenceFailed {
332        interrupted_turn_ids: Vec<TurnId>,
333        remaining_turn_ids: Vec<TurnId>,
334    },
335}
336
337#[derive(Clone)]
338struct QueuedTurnSteer {
339    message: UserMessage,
340    mailbox_ack: Option<MailboxDeliveryAck>,
341}
342
343#[derive(Clone)]
344struct MailboxDeliveryAck {
345    team_id: TeamId,
346    message_ids: Vec<String>,
347}
348
349#[derive(Clone)]
350struct InheritedTurnContext {
351    workspace: String,
352    instructions: InstructionBundle,
353    developer_context: Option<String>,
354}
355
356#[derive(Debug, Clone, Default, PartialEq, Eq)]
357pub struct ThreadActivity {
358    pub active_turn_id: Option<TurnId>,
359    pub active_flags: Vec<String>,
360}
361
362/// Per-thread settings persisted at thread creation and applied to every turn.
363#[derive(Debug, Clone, Default)]
364pub(crate) struct ThreadTurnOverrides {
365    pub(crate) tool_allowlist: Vec<String>,
366    pub(crate) developer_instructions: Option<String>,
367    pub(crate) external_tools: Vec<roder_api::tools::ToolSpec>,
368}
369
370#[derive(Debug, Clone, Copy, PartialEq, Eq)]
371pub(crate) enum TurnRunOutcome {
372    Completed,
373    Stopped,
374}
375
376impl PendingPlanExit {
377    pub fn new(
378        thread_id: ThreadId,
379        turn_id: TurnId,
380        request_id: String,
381        target_mode: PolicyMode,
382        plan_summary: Option<String>,
383        next_steps: Vec<String>,
384    ) -> Self {
385        let requested_at = OffsetDateTime::now_utc();
386        Self {
387            thread_id,
388            turn_id,
389            request_id,
390            target_mode,
391            plan_summary,
392            next_steps,
393            requested_at,
394            expires_at: Some(requested_at + default_plan_exit_timeout()),
395        }
396    }
397
398    pub fn is_expired(&self, now: OffsetDateTime) -> bool {
399        self.expires_at.is_some_and(|expires_at| now >= expires_at)
400    }
401}
402
403pub fn default_plan_exit_timeout() -> Duration {
404    Duration::minutes(10)
405}
406
407pub const DEFAULT_EXTERNAL_TOOL_TIMEOUT_SECONDS: u64 = 300;
408
409fn format_mailbox_messages(team: &TeamState, messages: &[TeamMailboxMessage]) -> String {
410    messages
411        .iter()
412        .map(|message| {
413            let recipient = team
414                .members
415                .iter()
416                .find(|member| member.id == message.to_member_id)
417                .map(canonical_team_member_path)
418                .unwrap_or_else(|| format!("/root/{}", message.to_member_id));
419            let sender = message
420                .from_member_id
421                .as_deref()
422                .and_then(|id| team.members.iter().find(|member| member.id == id))
423                .map(canonical_team_member_path)
424                .unwrap_or_else(|| "/root".to_string());
425            let message_type = match message.kind {
426                TeamMailboxMessageKind::Message => "MESSAGE",
427                TeamMailboxMessageKind::NewTask => "NEW_TASK",
428                TeamMailboxMessageKind::FinalAnswer => "FINAL_ANSWER",
429            };
430            format!(
431                "Message Type: {message_type}\nTask name: {recipient}\nSender: {sender}\nPayload:\n{}",
432                message.text
433            )
434        })
435        .collect::<Vec<_>>()
436        .join("\n\n")
437}
438
439fn canonical_team_member_path(member: &TeamMemberDescriptor) -> String {
440    if let Some(agent_path) = member
441        .agent_path
442        .as_deref()
443        .filter(|path| *path == "/root" || path.starts_with("/root/"))
444    {
445        return agent_path.to_string();
446    }
447    if member.role == TeamMemberRole::Lead {
448        return "/root".to_string();
449    }
450    member
451        .task_name
452        .as_deref()
453        .filter(|name| !name.trim().is_empty())
454        .map(|name| format!("/root/{}", name.trim().trim_matches('/')))
455        .unwrap_or_else(|| format!("/root/{}", member.id))
456}
457
458fn is_codex_v2_team(team: &TeamState) -> bool {
459    team.members.iter().any(|member| {
460        member
461            .model
462            .as_deref()
463            .is_some_and(|model| model_supports_reasoning_effort(model, REASONING_ULTRA))
464    })
465}
466
467fn runtime_local_team_view(
468    mut team: TeamState,
469    active_thread_ids: &std::collections::HashSet<ThreadId>,
470) -> TeamState {
471    for member in &mut team.members {
472        if member.status == TeamMemberStatus::Running
473            && !active_thread_ids.contains(&member.thread_id)
474        {
475            member.status = TeamMemberStatus::Interrupted;
476            member.current_turn_id = None;
477        }
478    }
479    team
480}
481
482type RunnerToolExecutionKey = (String, String);
483type RunnerToolExecutionMutex = Mutex<()>;
484type RunnerToolExecutionLockRegistry =
485    Mutex<HashMap<RunnerToolExecutionKey, Weak<RunnerToolExecutionMutex>>>;
486
487pub struct Runtime {
488    pub bus: EventBus,
489    pub registry: ExtensionRegistry,
490    config: RwLock<RuntimeConfig>,
491    pending_plan_exit: RwLock<Option<PendingPlanExit>>,
492    pub(crate) pending_tool_approvals: Mutex<HashMap<String, PendingToolApproval>>,
493    pub(crate) pending_user_inputs: Mutex<HashMap<String, PendingUserInput>>,
494    pub(crate) pending_external_tool_calls: Mutex<HashMap<String, PendingExternalToolCall>>,
495    /// Serializes turn admission with shutdown quiescing. A drain takes this
496    /// gate before flipping `accepting_turns` and snapshotting active work so a
497    /// turn that observed the old flag cannot be inserted after the snapshot.
498    turn_admission: Mutex<()>,
499    active_turns: RwLock<HashMap<TurnId, ActiveTurnHandle>>,
500    turn_drains: RwLock<HashMap<TurnId, Arc<TurnDrainHandle>>>,
501    /// Provider cleanup handles register synchronously through the inference
502    /// tool-executor context. They are intentionally separate from active turn
503    /// admission so an interrupted turn can remain drainable after a follow-up
504    /// turn starts on the same thread.
505    provider_turn_cleanups: std::sync::Mutex<HashMap<TurnId, Arc<dyn ProviderTurnCleanup>>>,
506    accepting_turns: AtomicBool,
507    /// Monotonic counter used by bounded drains to surface lifecycle-state
508    /// persistence failures without making an individual provider turn fail.
509    lifecycle_persistence_failures: AtomicUsize,
510    lifecycle_shutdown_drains: AtomicUsize,
511    lifecycle_clean_shutdowns: AtomicUsize,
512    lifecycle_deadline_exceeded: AtomicUsize,
513    lifecycle_persistence_failed_drains: AtomicUsize,
514    lifecycle_restart_reconciliations: AtomicUsize,
515    lifecycle_shutdown_drain_duration_ms_total: AtomicUsize,
516    provider_cleanup_confirmed: AtomicUsize,
517    provider_cleanup_timed_out: AtomicUsize,
518    provider_cleanup_unknown: AtomicUsize,
519    active_turns_changed: Notify,
520    active_turn_selections: RwLock<HashMap<TurnId, ModelSelectionMode>>,
521    active_turn_contexts: RwLock<HashMap<TurnId, InheritedTurnContext>>,
522    /// Process-local execution boundary. Local/CLI runtimes default to true;
523    /// hosted runtime pools set this false so missing runner metadata fails
524    /// closed instead of exposing the host workspace.
525    allow_local_workspaces: AtomicBool,
526    // Spawn-time live authority retained for every reusable turn of a long-lived teammate.
527    // This stays process-local because developer_context is explicitly volatile and must never
528    // be persisted to thread state.
529    team_member_turn_contexts: Mutex<HashMap<ThreadId, InheritedTurnContext>>,
530    workspace: PathBuf,
531    teams: TeamManager,
532    agent_team_spawn_lock: Mutex<()>,
533    pub(crate) roadmaps: Mutex<roder_roadmap::RoadmapRuntime>,
534    /// Completed reviews, most recent last, so a later publish can resolve a
535    /// review id back to its findings. Bounded; see `review::run`.
536    pub(crate) reviews: Mutex<Vec<crate::review::ReviewRecord>>,
537    pub(crate) goals: Arc<RuntimeGoalController>,
538    context_artifacts: roder_api::artifacts::ContextArtifactStore,
539    pub(crate) thread_store: Option<Arc<dyn ThreadStore>>,
540    thread_item_cache: Mutex<ThreadItemCache>,
541    pub(crate) tool_registry: ToolRegistry,
542    media_generation: Arc<crate::media_generation::MediaGenerationService>,
543    pub(crate) skills: RwLock<SkillRegistry>,
544    /// Lazily-started bounded dispatch of emitted events to registry
545    /// `EventSink`s (process extensions etc.); see `event_sink_dispatch`.
546    event_sink_dispatcher: tokio::sync::OnceCell<crate::event_sink_dispatch::EventSinkDispatcher>,
547    pub(crate) compaction_hysteresis: std::sync::Mutex<HashMap<ThreadId, u32>>,
548    /// Per-thread agent-swarm mode overrides (roadmap 104). A thread present in
549    /// this map uses its stored value; absent threads fall back to the
550    /// runtime-global `RuntimeConfig.agent_swarm_mode` default. This mirrors the
551    /// team per-member policy-mode override idiom so swarm mode is per-thread
552    /// like the other `thread/*` operations, without a separate runtime.
553    agent_swarm_modes: RwLock<HashMap<ThreadId, bool>>,
554    /**
555     * Live remote-runner sessions keyed by thread. Each per-thread mutex is
556     * held across provider initialization, making the first workspace-tool
557     * access a singleflight while allowing unrelated threads to initialize in
558     * parallel. Failed initialization is shared with current waiters, then
559     * evicted so a later tool call can retry.
560     */
561    runner_sessions: Arc<Mutex<HashMap<ThreadId, Arc<RunnerSessionSlot>>>>,
562    /**
563     * Outer tool-call locks keyed by the actual remote session, not the
564     * per-thread destination id. Hosted runtimes are tenant-scoped, so this
565     * serializes threads that share one sandbox without coupling different
566     * tenants or independent runner sessions.
567     */
568    runner_tool_execution_locks: RunnerToolExecutionLockRegistry,
569}
570
571#[derive(Clone)]
572struct CachedRunnerSession {
573    destination: RunnerDestination,
574    session: Arc<dyn RemoteRunnerSession>,
575}
576
577struct RunnerSessionSlot {
578    provider_id: String,
579    destination_id: String,
580    state: Mutex<RunnerSessionSlotState>,
581    initialized: Notify,
582}
583
584enum RunnerSessionSlotState {
585    Empty,
586    Initializing,
587    Ready(CachedRunnerSession),
588    Failed(Arc<str>),
589}
590
591impl RunnerSessionSlot {
592    fn empty(destination: &RunnerDestination) -> Self {
593        Self {
594            provider_id: destination.provider_id.clone(),
595            destination_id: destination.id.clone(),
596            state: Mutex::new(RunnerSessionSlotState::Empty),
597            initialized: Notify::new(),
598        }
599    }
600
601    fn ready(session: CachedRunnerSession) -> Self {
602        Self {
603            provider_id: session.destination.provider_id.clone(),
604            destination_id: session.destination.id.clone(),
605            state: Mutex::new(RunnerSessionSlotState::Ready(session)),
606            initialized: Notify::new(),
607        }
608    }
609
610    fn matches(&self, destination: &RunnerDestination) -> bool {
611        self.provider_id == destination.provider_id && self.destination_id == destination.id
612    }
613}
614
615impl Runtime {
616    pub fn new(registry: ExtensionRegistry, config: RuntimeConfig) -> anyhow::Result<Self> {
617        if registry.inference_engines.is_empty() {
618            anyhow::bail!("at least one inference engine must be registered");
619        }
620        validate_runtime_config_reasoning(&config)?;
621        validate_runtime_inference_router_config(&registry, &config)?;
622
623        let bus = EventBus::new(EVENT_BUS_CAPACITY);
624        let thread_store = registry
625            .thread_stores
626            .first()
627            .map(|factory| factory.create());
628        let mut tool_registry = ToolRegistry::default();
629        for contributor in &registry.tools {
630            contributor
631                .contribute(&mut tool_registry)
632                .with_context(|| format!("tool contributor {} failed", contributor.id()))?;
633        }
634        crate::agent_control_tools::contribute_agent_control_tools(&mut tool_registry)?;
635
636        let media_generation = Arc::new(crate::media_generation::MediaGenerationService::new(
637            registry.media_generator_providers.clone(),
638            config.media_generation.clone(),
639        ));
640        tool_registry.replace(Arc::new(
641            crate::media_generation::MediaGenerateImageTool::new(media_generation.clone()),
642        ));
643
644        let team_data_dir = config.team_data_dir.clone();
645        let workspace = config
646            .workspace
647            .clone()
648            .map(PathBuf::from)
649            .unwrap_or(std::env::current_dir()?);
650        let roadmap_data_dir = config
651            .roadmap_data_dir
652            .clone()
653            .unwrap_or_else(|| workspace.join(".roder"));
654        let context_artifacts = thread_store
655            .as_ref()
656            .and_then(|store| store.context_artifact_store())
657            .or_else(|| {
658                thread_store
659                    .as_ref()
660                    .and_then(|store| store.local_thread_root())
661                    .map(FilesystemContextArtifactStore::shared_thread_scoped)
662            })
663            .unwrap_or_else(|| {
664                FilesystemContextArtifactStore::shared_legacy(default_context_artifact_dir())
665            });
666        let goals = Arc::new(RuntimeGoalController::new(
667            bus.clone(),
668            thread_store.clone(),
669        ));
670        let runtime = Self {
671            bus,
672            registry,
673            config: RwLock::new(config),
674            pending_plan_exit: RwLock::new(None),
675            pending_tool_approvals: Mutex::new(HashMap::new()),
676            pending_user_inputs: Mutex::new(HashMap::new()),
677            pending_external_tool_calls: Mutex::new(HashMap::new()),
678            turn_admission: Mutex::new(()),
679            active_turns: RwLock::new(HashMap::new()),
680            turn_drains: RwLock::new(HashMap::new()),
681            provider_turn_cleanups: std::sync::Mutex::new(HashMap::new()),
682            accepting_turns: AtomicBool::new(true),
683            lifecycle_persistence_failures: AtomicUsize::new(0),
684            lifecycle_shutdown_drains: AtomicUsize::new(0),
685            lifecycle_clean_shutdowns: AtomicUsize::new(0),
686            lifecycle_deadline_exceeded: AtomicUsize::new(0),
687            lifecycle_persistence_failed_drains: AtomicUsize::new(0),
688            lifecycle_restart_reconciliations: AtomicUsize::new(0),
689            lifecycle_shutdown_drain_duration_ms_total: AtomicUsize::new(0),
690            provider_cleanup_confirmed: AtomicUsize::new(0),
691            provider_cleanup_timed_out: AtomicUsize::new(0),
692            provider_cleanup_unknown: AtomicUsize::new(0),
693            active_turns_changed: Notify::new(),
694            active_turn_selections: RwLock::new(HashMap::new()),
695            active_turn_contexts: RwLock::new(HashMap::new()),
696            allow_local_workspaces: AtomicBool::new(true),
697            team_member_turn_contexts: Mutex::new(HashMap::new()),
698            workspace: workspace.clone(),
699            teams: TeamManager::new(
700                team_data_dir.unwrap_or_else(crate::teams::default_team_data_dir),
701            ),
702            agent_team_spawn_lock: Mutex::new(()),
703            roadmaps: Mutex::new(roder_roadmap::RoadmapRuntime::new(
704                workspace,
705                roadmap_data_dir,
706            )),
707            reviews: Mutex::new(Vec::new()),
708            goals,
709            context_artifacts,
710            thread_store,
711            thread_item_cache: Mutex::new(ThreadItemCache::default()),
712            tool_registry,
713            media_generation,
714            skills: RwLock::new(SkillRegistry::load(SkillRegistryOptions::new(
715                PathBuf::new(),
716            ))),
717            event_sink_dispatcher: tokio::sync::OnceCell::new(),
718            compaction_hysteresis: crate::compaction_runtime::compaction_hysteresis_state(),
719            agent_swarm_modes: RwLock::new(HashMap::new()),
720            runner_sessions: Arc::new(Mutex::new(HashMap::new())),
721            runner_tool_execution_locks: Mutex::new(HashMap::new()),
722        };
723        runtime.bus.emit(RoderEvent::RuntimeStarted(RuntimeStarted {
724            timestamp: OffsetDateTime::now_utc(),
725        }));
726        for manifest in &runtime.registry.manifests {
727            runtime
728                .bus
729                .emit(RoderEvent::ExtensionRegistered(ExtensionRegistered {
730                    extension_id: manifest.id.clone(),
731                    timestamp: OffsetDateTime::now_utc(),
732                }));
733        }
734        Ok(runtime)
735    }
736
737    pub fn from_engine(engine: Arc<dyn InferenceEngine>) -> anyhow::Result<Self> {
738        let mut builder = roder_api::extension::ExtensionRegistryBuilder::new();
739        builder.inference_engine(engine);
740        Self::new(builder.build()?, RuntimeConfig::default())
741    }
742
743    pub fn fake() -> anyhow::Result<Self> {
744        Self::from_engine(Arc::new(FakeInferenceEngine))
745    }
746
747    pub fn subscribe_events(&self) -> tokio::sync::broadcast::Receiver<EventEnvelope> {
748        self.bus.subscribe()
749    }
750
751    /// Returns process-local, redacted lifecycle health counters. The snapshot
752    /// intentionally has fixed fields and contains no provider, command,
753    /// process, thread, or turn identifiers.
754    pub fn lifecycle_metrics(&self) -> LifecycleMetricsSnapshot {
755        LifecycleMetricsSnapshot {
756            shutdown_drain_count: self.lifecycle_shutdown_drains.load(Ordering::Acquire) as u64,
757            clean_shutdown_count: self.lifecycle_clean_shutdowns.load(Ordering::Acquire) as u64,
758            deadline_exceeded_count: self.lifecycle_deadline_exceeded.load(Ordering::Acquire)
759                as u64,
760            persistence_failed_count: self
761                .lifecycle_persistence_failed_drains
762                .load(Ordering::Acquire) as u64,
763            restart_reconciliation_count: self
764                .lifecycle_restart_reconciliations
765                .load(Ordering::Acquire) as u64,
766            lifecycle_persistence_failure_count: self
767                .lifecycle_persistence_failures
768                .load(Ordering::Acquire) as u64,
769            shutdown_drain_duration_ms_total: self
770                .lifecycle_shutdown_drain_duration_ms_total
771                .load(Ordering::Acquire) as u64,
772            provider_cleanup_confirmed_count: self
773                .provider_cleanup_confirmed
774                .load(Ordering::Acquire) as u64,
775            provider_cleanup_timed_out_count: self
776                .provider_cleanup_timed_out
777                .load(Ordering::Acquire) as u64,
778            provider_cleanup_unknown_count: self.provider_cleanup_unknown.load(Ordering::Acquire)
779                as u64,
780        }
781    }
782
783    async fn persist_turn_lifecycle_record(&self, record: &TurnLifecycleRecord) -> bool {
784        let Some(store) = &self.thread_store else {
785            return true;
786        };
787        let state = match record.extension_state() {
788            Ok(state) => state,
789            Err(_) => {
790                self.lifecycle_persistence_failures
791                    .fetch_add(1, Ordering::AcqRel);
792                return false;
793            }
794        };
795        // Lifecycle diagnostics must not turn a provider result into a failed
796        // turn merely because a third-party store lacks extension-state support.
797        // JSONL/Postgres stores persist this append-only record atomically at
798        // their own storage boundary.
799        if store
800            .append_extension_state(&record.thread_id, &state)
801            .await
802            .is_err()
803        {
804            self.lifecycle_persistence_failures
805                .fetch_add(1, Ordering::AcqRel);
806            return false;
807        }
808        true
809    }
810
811    async fn record_turn_lifecycle(
812        &self,
813        thread_id: ThreadId,
814        turn_id: TurnId,
815        state: TurnLifecycleState,
816        cleanup: TurnCleanupState,
817        reason: Option<TurnLifecycleReason>,
818    ) -> TurnLifecycleRecord {
819        self.record_turn_lifecycle_with_ownership(
820            thread_id,
821            turn_id,
822            state,
823            cleanup,
824            reason,
825            TurnCleanupOwnership::RuntimeTaskOnly,
826        )
827        .await
828    }
829
830    async fn record_turn_lifecycle_with_ownership(
831        &self,
832        thread_id: ThreadId,
833        turn_id: TurnId,
834        state: TurnLifecycleState,
835        cleanup: TurnCleanupState,
836        reason: Option<TurnLifecycleReason>,
837        ownership: TurnCleanupOwnership,
838    ) -> TurnLifecycleRecord {
839        let record = TurnLifecycleRecord::new(
840            thread_id,
841            turn_id,
842            state,
843            cleanup,
844            reason,
845            OffsetDateTime::now_utc(),
846        )
847        .with_ownership(ownership);
848        let _ = self.persist_turn_lifecycle_record(&record).await;
849        self.emit(RoderEvent::TurnLifecycleUpdated(record.clone()))
850            .await;
851        record
852    }
853
854    pub(crate) fn register_provider_turn_cleanup(
855        &self,
856        turn_id: &TurnId,
857        cleanup: Arc<dyn ProviderTurnCleanup>,
858    ) {
859        if let Ok(mut cleanups) = self.provider_turn_cleanups.lock() {
860            cleanups.insert(turn_id.clone(), cleanup);
861        }
862    }
863
864    fn provider_cleanup_ownership(&self, turn_id: &TurnId) -> TurnCleanupOwnership {
865        self.provider_turn_cleanups
866            .lock()
867            .ok()
868            .and_then(|cleanups| cleanups.get(turn_id).cloned())
869            .map(|cleanup| cleanup.ownership())
870            .unwrap_or(TurnCleanupOwnership::RuntimeTaskOnly)
871    }
872
873    async fn await_provider_turn_cleanup(
874        &self,
875        turn_id: &TurnId,
876    ) -> (TurnCleanupState, TurnCleanupOwnership) {
877        let cleanup = self
878            .provider_turn_cleanups
879            .lock()
880            .ok()
881            .and_then(|mut cleanups| cleanups.remove(turn_id));
882        let Some(cleanup) = cleanup else {
883            return (
884                TurnCleanupState::Unknown,
885                TurnCleanupOwnership::RuntimeTaskOnly,
886            );
887        };
888        let ownership = cleanup.ownership();
889        match tokio::time::timeout(PROVIDER_CLEANUP_TIMEOUT, cleanup.wait_for_cleanup()).await {
890            Ok(Ok(())) => {
891                self.provider_cleanup_confirmed
892                    .fetch_add(1, Ordering::AcqRel);
893                (
894                    TurnCleanupState::Completed,
895                    TurnCleanupOwnership::ProviderCleanupConfirmed,
896                )
897            }
898            Ok(Err(_)) => {
899                self.provider_cleanup_unknown.fetch_add(1, Ordering::AcqRel);
900                (TurnCleanupState::Unknown, ownership)
901            }
902            Err(_) => {
903                self.provider_cleanup_timed_out
904                    .fetch_add(1, Ordering::AcqRel);
905                (TurnCleanupState::TimedOut, ownership)
906            }
907        }
908    }
909
910    fn record_lifecycle_drain_outcome(
911        &self,
912        started_at: tokio::time::Instant,
913        outcome: &RuntimeDrainOutcome,
914    ) {
915        self.lifecycle_shutdown_drains
916            .fetch_add(1, Ordering::AcqRel);
917        self.lifecycle_shutdown_drain_duration_ms_total.fetch_add(
918            started_at.elapsed().as_millis().min(usize::MAX as u128) as usize,
919            Ordering::AcqRel,
920        );
921        match outcome {
922            RuntimeDrainOutcome::Clean { .. } => {
923                self.lifecycle_clean_shutdowns
924                    .fetch_add(1, Ordering::AcqRel);
925            }
926            RuntimeDrainOutcome::DeadlineExceeded { .. } => {
927                self.lifecycle_deadline_exceeded
928                    .fetch_add(1, Ordering::AcqRel);
929            }
930            RuntimeDrainOutcome::PersistenceFailed { .. } => {
931                self.lifecycle_persistence_failed_drains
932                    .fetch_add(1, Ordering::AcqRel);
933            }
934        }
935    }
936
937    fn lifecycle_record_for_event(event: &RoderEvent) -> Option<TurnLifecycleRecord> {
938        match event {
939            RoderEvent::TurnStarted(event) => Some(TurnLifecycleRecord::new(
940                event.thread_id.clone(),
941                event.turn_id.clone(),
942                TurnLifecycleState::Running,
943                TurnCleanupState::NotRequested,
944                None,
945                event.timestamp,
946            )),
947            // `run_turn` persists completed turns after it has awaited a
948            // registered provider cleanup handle. Do not let the generic event
949            // projection overwrite that acknowledgement with an unproven state.
950            RoderEvent::TurnCompleted(_) => None,
951            // Some failure paths already have an event before the turn wrapper
952            // reaches its cleanup acknowledgement. Preserve a durable fallback
953            // reason here; the wrapper appends a later record with the more
954            // precise cleanup outcome when a provider registered one.
955            RoderEvent::TurnFailed(event) => Some(TurnLifecycleRecord::new(
956                event.thread_id.clone(),
957                event.turn_id.clone(),
958                TurnLifecycleState::Failed,
959                TurnCleanupState::Unknown,
960                Some(if event.error_kind.as_deref() == Some("deadline_timeout") {
961                    TurnLifecycleReason::DeadlineExceeded
962                } else {
963                    TurnLifecycleReason::ProviderFailure
964                }),
965                event.timestamp,
966            )),
967            RoderEvent::TurnDeadlineExceeded(_) | RoderEvent::TurnInterrupted(_) => None,
968            RoderEvent::TurnLifecycleUpdated(_) => None,
969            _ => None,
970        }
971    }
972
973    pub fn registry(&self) -> &ExtensionRegistry {
974        &self.registry
975    }
976
977    pub fn media_generation(&self) -> Arc<crate::media_generation::MediaGenerationService> {
978        self.media_generation.clone()
979    }
980
981    pub fn context_artifacts(&self) -> roder_api::artifacts::ContextArtifactStore {
982        self.context_artifacts.clone()
983    }
984
985    pub async fn execute_workflow_tool(
986        &self,
987        thread_id: ThreadId,
988        tool_name: &str,
989        arguments: serde_json::Value,
990    ) -> anyhow::Result<ToolResult> {
991        let Some(executor) = self.tool_registry.get(tool_name) else {
992            anyhow::bail!("tool not found: {tool_name}");
993        };
994        let tool_call = ToolCall {
995            id: format!("slash-{tool_name}"),
996            name: tool_name.to_string(),
997            raw_arguments: serde_json::to_string(&arguments)?,
998            arguments,
999            thread_id: thread_id.clone(),
1000            turn_id: "slash-command".to_string(),
1001        };
1002        let runtime_config = self.status().await;
1003        let ctx = self.tool_execution_context(
1004            thread_id,
1005            "slash-command".to_string(),
1006            runtime_config.policy_mode,
1007            runtime_config.workspace.as_deref(),
1008            Some(&runtime_config.command_shell),
1009        );
1010        executor.execute(ctx, tool_call).await
1011    }
1012
1013    pub(crate) fn tool_execution_context(
1014        &self,
1015        thread_id: ThreadId,
1016        turn_id: TurnId,
1017        mode: PolicyMode,
1018        workspace: Option<&str>,
1019        command_shell: Option<&str>,
1020    ) -> ToolExecutionContext {
1021        let mut ctx = ToolExecutionContext::new(thread_id, turn_id, mode)
1022            .with_command_shell(command_shell.unwrap_or_default())
1023            .with_process_runner(Arc::new(LocalProcessRunner))
1024            .with_context_artifacts(self.context_artifacts.backend())
1025            .with_goal_controller(self.goals.clone())
1026            .with_subagent_trace_sink(Arc::new(RuntimeSubagentTraceSink::new(
1027                self.bus.clone(),
1028                self.thread_store.clone(),
1029            )))
1030            .with_swarm_progress_sink(Arc::new(RuntimeAgentSwarmProgressSink::new(
1031                self.bus.clone(),
1032                self.thread_store.clone(),
1033            )));
1034        if let Some(workspace) = workspace {
1035            ctx = ctx.with_workspace_handle(Arc::new(ScopedFilesystem::new(workspace)));
1036        }
1037        ctx
1038    }
1039
1040    pub async fn status(&self) -> RuntimeConfig {
1041        self.config.read().await.clone()
1042    }
1043
1044    pub async fn set_skills(&self, skills: SkillRegistry) {
1045        *self.skills.write().await = skills;
1046    }
1047
1048    pub async fn skills_snapshot(&self) -> SkillRegistry {
1049        self.skills.read().await.clone()
1050    }
1051
1052    pub fn workspace(&self) -> PathBuf {
1053        self.workspace.clone()
1054    }
1055
1056    /// Controls whether native workspace tools may execute without an
1057    /// explicit remote-runner binding. Hosted runtimes disable this after
1058    /// construction; ordinary local runtimes retain the compatible default.
1059    pub fn set_allow_local_workspaces(&self, allowed: bool) {
1060        self.allow_local_workspaces
1061            .store(allowed, Ordering::Relaxed);
1062    }
1063
1064    pub fn allows_local_workspaces(&self) -> bool {
1065        self.allow_local_workspaces.load(Ordering::Relaxed)
1066    }
1067
1068    pub async fn set_remote_runner_destination(&self, destination: Option<RunnerDestination>) {
1069        let lifecycle = destination.as_ref().map(|destination| RunnerLifecycle {
1070            destination_id: destination.id.clone(),
1071            provider_id: destination.provider_id.clone(),
1072            state: "configured".to_string(),
1073            session_id: None,
1074            timestamp: OffsetDateTime::now_utc(),
1075        });
1076        self.config.write().await.remote_runner_destination = destination;
1077        if let Some(lifecycle) = lifecycle {
1078            self.emit(RoderEvent::RunnerLifecycle(lifecycle)).await;
1079        } else {
1080            self.emit(RoderEvent::RunnerLifecycle(RunnerLifecycle {
1081                destination_id: "local".to_string(),
1082                provider_id: "local".to_string(),
1083                state: "local_fallback".to_string(),
1084                session_id: None,
1085                timestamp: OffsetDateTime::now_utc(),
1086            }))
1087            .await;
1088        }
1089    }
1090
1091    pub async fn set_file_backed_dynamic_context(&self, enabled: bool) -> RuntimeConfig {
1092        let mut cfg = self.config.write().await;
1093        cfg.file_backed_dynamic_context = enabled;
1094        cfg.clone()
1095    }
1096
1097    pub async fn set_command_shell(&self, shell: String) -> RuntimeConfig {
1098        let mut cfg = self.config.write().await;
1099        cfg.command_shell = shell;
1100        cfg.clone()
1101    }
1102
1103    pub async fn pending_plan_exit(&self) -> Option<PendingPlanExit> {
1104        let mut pending = self.pending_plan_exit.write().await;
1105        let current = pending.clone()?;
1106        if !current.is_expired(OffsetDateTime::now_utc()) {
1107            return Some(current);
1108        }
1109        *pending = None;
1110        drop(pending);
1111        self.emit_plan_exit_resolved(&current, false, self.status().await.policy_mode)
1112            .await;
1113        None
1114    }
1115
1116    pub async fn set_policy_mode(
1117        &self,
1118        mode: PolicyMode,
1119        reason: Option<String>,
1120    ) -> anyhow::Result<RuntimeConfig> {
1121        let mut cfg = self.config.write().await;
1122        let previous_mode = cfg.policy_mode;
1123        cfg.policy_mode = mode;
1124        let next = cfg.clone();
1125        drop(cfg);
1126        self.emit(RoderEvent::PolicyModeChanged(PolicyModeChanged {
1127            thread_id: "runtime".to_string(),
1128            turn_id: None,
1129            previous_mode,
1130            new_mode: mode,
1131            reason,
1132            timestamp: OffsetDateTime::now_utc(),
1133        }))
1134        .await;
1135        self.auto_resolve_pending_tool_approvals_for_mode(mode)
1136            .await;
1137        Ok(next)
1138    }
1139
1140    /// Toggle agent-swarm mode for the runtime (roadmap 104). When enabled, the
1141    /// swarm reminder is injected into each turn's developer instructions so
1142    /// every app-server/SDK client gets it, not only the TUI.
1143    pub async fn set_agent_swarm_mode(
1144        &self,
1145        enabled: bool,
1146        trigger: roder_api::subagents::AgentSwarmModeTrigger,
1147    ) -> anyhow::Result<RuntimeConfig> {
1148        let mut cfg = self.config.write().await;
1149        cfg.agent_swarm_mode = enabled;
1150        let next = cfg.clone();
1151        drop(cfg);
1152        self.emit(RoderEvent::AgentSwarmModeChanged(
1153            roder_api::subagents::AgentSwarmModeChanged {
1154                thread_id: "runtime".to_string(),
1155                turn_id: None,
1156                enabled,
1157                trigger,
1158                timestamp: OffsetDateTime::now_utc(),
1159            },
1160        ))
1161        .await;
1162        Ok(next)
1163    }
1164
1165    /// Toggle agent-swarm mode for a single thread (roadmap 104). The override is
1166    /// stored per thread so toggling swarm mode on one thread does not leak the
1167    /// reminder into other threads sharing the runtime; absent threads fall back
1168    /// to the runtime-global default. The emitted `AgentSwarmModeChanged` event
1169    /// carries the real `thread_id`.
1170    pub async fn set_agent_swarm_mode_for_thread(
1171        &self,
1172        thread_id: &str,
1173        enabled: bool,
1174        trigger: roder_api::subagents::AgentSwarmModeTrigger,
1175    ) -> bool {
1176        {
1177            let mut modes = self.agent_swarm_modes.write().await;
1178            modes.insert(thread_id.to_string(), enabled);
1179        }
1180        self.emit(RoderEvent::AgentSwarmModeChanged(
1181            roder_api::subagents::AgentSwarmModeChanged {
1182                thread_id: thread_id.to_string(),
1183                turn_id: None,
1184                enabled,
1185                trigger,
1186                timestamp: OffsetDateTime::now_utc(),
1187            },
1188        ))
1189        .await;
1190        enabled
1191    }
1192
1193    /// Resolve whether agent-swarm mode is active for `thread_id`: the per-thread
1194    /// override when present, otherwise the runtime-global default. The turn loop
1195    /// uses this to decide whether to inject the swarm reminder.
1196    pub async fn effective_agent_swarm_mode_for_thread(&self, thread_id: &str) -> bool {
1197        if let Some(enabled) = self.agent_swarm_modes.read().await.get(thread_id).copied() {
1198            return enabled;
1199        }
1200        self.status().await.agent_swarm_mode
1201    }
1202
1203    pub async fn set_hosted_web_search(
1204        &self,
1205        mode: HostedWebSearchMode,
1206    ) -> anyhow::Result<RuntimeConfig> {
1207        let mut cfg = self.config.write().await;
1208        cfg.hosted_web_search = HostedWebSearchConfig { mode };
1209        Ok(cfg.clone())
1210    }
1211
1212    async fn auto_resolve_pending_tool_approvals_for_mode(&self, mode: PolicyMode) {
1213        let gate = DefaultPolicyGate::new();
1214        let mut pending = self.pending_tool_approvals.lock().await;
1215        let approval_ids = pending
1216            .iter()
1217            .filter_map(|(approval_id, approval)| {
1218                let ctx = ToolExecutionContext::new(
1219                    approval.thread_id.clone(),
1220                    approval.turn_id.clone(),
1221                    mode,
1222                );
1223                matches!(
1224                    gate.decide(&approval.call, mode, &ctx),
1225                    PolicyDecision::AutoApproved { .. }
1226                )
1227                .then_some(approval_id.clone())
1228            })
1229            .collect::<Vec<_>>();
1230        let approvals = approval_ids
1231            .into_iter()
1232            .filter_map(|approval_id| {
1233                pending
1234                    .remove(&approval_id)
1235                    .map(|approval| (approval_id, approval))
1236            })
1237            .collect::<Vec<_>>();
1238        drop(pending);
1239
1240        for (approval_id, approval) in approvals {
1241            let ctx = ToolExecutionContext::new(
1242                approval.thread_id.clone(),
1243                approval.turn_id.clone(),
1244                mode,
1245            );
1246            let decision = gate.decide(&approval.call, mode, &ctx);
1247            self.emit(RoderEvent::PolicyDecisionRecorded(PolicyDecisionRecorded {
1248                thread_id: approval.thread_id.clone(),
1249                turn_id: approval.turn_id.clone(),
1250                tool_id: approval.tool_id.clone(),
1251                tool_name: approval.tool_name.clone(),
1252                mode,
1253                decision,
1254                timestamp: OffsetDateTime::now_utc(),
1255            }))
1256            .await;
1257            if mode == PolicyMode::Bypass {
1258                self.emit(RoderEvent::PolicyBypassActive(PolicyBypassActive {
1259                    thread_id: approval.thread_id.clone(),
1260                    turn_id: approval.turn_id.clone(),
1261                    tool_id: approval.tool_id.clone(),
1262                    tool_name: approval.tool_name.clone(),
1263                    timestamp: OffsetDateTime::now_utc(),
1264                }))
1265                .await;
1266            }
1267            self.emit(RoderEvent::ApprovalResolved(ApprovalResolved {
1268                thread_id: approval.thread_id,
1269                turn_id: approval.turn_id,
1270                approval_id,
1271                tool_id: approval.tool_id,
1272                tool_name: approval.tool_name,
1273                approved: true,
1274                timestamp: OffsetDateTime::now_utc(),
1275            }))
1276            .await;
1277            let _ = approval.tx.send(true);
1278        }
1279    }
1280
1281    pub async fn record_pending_plan_exit(&self, pending: PendingPlanExit) {
1282        *self.pending_plan_exit.write().await = Some(pending.clone());
1283        self.emit(RoderEvent::PolicyExitPlanRequested(
1284            PolicyExitPlanRequested {
1285                thread_id: pending.thread_id,
1286                turn_id: pending.turn_id,
1287                request_id: pending.request_id,
1288                target_mode: pending.target_mode,
1289                plan_summary: pending.plan_summary,
1290                next_steps: pending.next_steps,
1291                timestamp: OffsetDateTime::now_utc(),
1292            },
1293        ))
1294        .await;
1295    }
1296
1297    pub async fn resolve_pending_plan_exit(
1298        &self,
1299        request_id: &str,
1300        approved: bool,
1301    ) -> anyhow::Result<Option<PendingPlanExit>> {
1302        let mut pending = self.pending_plan_exit.write().await;
1303        let Some(current) = pending.clone() else {
1304            return Ok(None);
1305        };
1306        if current.request_id != request_id {
1307            anyhow::bail!("pending plan exit request {request_id:?} was not found");
1308        }
1309        *pending = None;
1310        drop(pending);
1311
1312        let approved = approved && !current.is_expired(OffsetDateTime::now_utc());
1313        let resolved_mode = if approved {
1314            let mut cfg = self.config.write().await;
1315            let previous_mode = cfg.policy_mode;
1316            cfg.policy_mode = current.target_mode;
1317            drop(cfg);
1318            self.emit(RoderEvent::PolicyModeChanged(PolicyModeChanged {
1319                thread_id: current.thread_id.clone(),
1320                turn_id: Some(current.turn_id.clone()),
1321                previous_mode,
1322                new_mode: current.target_mode,
1323                reason: Some("approved plan exit".to_string()),
1324                timestamp: OffsetDateTime::now_utc(),
1325            }))
1326            .await;
1327            self.auto_resolve_pending_tool_approvals_for_mode(current.target_mode)
1328                .await;
1329            current.target_mode
1330        } else {
1331            self.status().await.policy_mode
1332        };
1333        self.emit_plan_exit_resolved(&current, approved, resolved_mode)
1334            .await;
1335        Ok(Some(current))
1336    }
1337
1338    pub async fn resolve_tool_approval(
1339        &self,
1340        approval_id: &str,
1341        approved: bool,
1342    ) -> anyhow::Result<bool> {
1343        let pending = self.pending_tool_approvals.lock().await.remove(approval_id);
1344        let Some(pending) = pending else {
1345            return Ok(false);
1346        };
1347        self.emit(RoderEvent::ApprovalResolved(ApprovalResolved {
1348            thread_id: pending.thread_id,
1349            turn_id: pending.turn_id,
1350            approval_id: approval_id.to_string(),
1351            tool_id: pending.tool_id,
1352            tool_name: pending.tool_name,
1353            approved,
1354            timestamp: OffsetDateTime::now_utc(),
1355        }))
1356        .await;
1357        let _ = pending.tx.send(approved);
1358        Ok(true)
1359    }
1360
1361    pub async fn request_app_server_tool_approval(
1362        &self,
1363        call: ToolCall,
1364        reason: Option<String>,
1365    ) -> anyhow::Result<bool> {
1366        let approval_id = call.id.clone();
1367        let (tx, rx) = oneshot::channel();
1368        self.pending_tool_approvals.lock().await.insert(
1369            approval_id.clone(),
1370            PendingToolApproval {
1371                thread_id: call.thread_id.clone(),
1372                turn_id: call.turn_id.clone(),
1373                tool_id: call.id.clone(),
1374                tool_name: call.name.clone(),
1375                call: call.clone(),
1376                tx,
1377            },
1378        );
1379        self.emit(RoderEvent::ApprovalRequested(ApprovalRequested {
1380            thread_id: call.thread_id.clone(),
1381            turn_id: call.turn_id.clone(),
1382            approval_id,
1383            tool_id: call.id.clone(),
1384            tool_name: call.name.clone(),
1385            reason,
1386            timestamp: OffsetDateTime::now_utc(),
1387        }))
1388        .await;
1389        Ok(rx.await.unwrap_or(false))
1390    }
1391
1392    /// Completes a pending host-executed tool call (`tools/resolve`). Returns false when the
1393    /// request id is unknown, already resolved, timed out, or cancelled by a turn interrupt.
1394    pub async fn resolve_external_tool_call(
1395        &self,
1396        request_id: &str,
1397        resolution: ExternalToolResolution,
1398    ) -> anyhow::Result<bool> {
1399        let pending = self
1400            .pending_external_tool_calls
1401            .lock()
1402            .await
1403            .remove(request_id);
1404        let Some(pending) = pending else {
1405            return Ok(false);
1406        };
1407        self.emit(RoderEvent::ExternalToolCallResolved(
1408            ExternalToolCallResolved {
1409                thread_id: pending.thread_id,
1410                turn_id: pending.turn_id,
1411                request_id: request_id.to_string(),
1412                tool_id: pending.tool_id,
1413                tool_name: pending.tool_name,
1414                outcome: ExternalToolCallOutcome::Resolved,
1415                is_error: resolution.is_error,
1416                timestamp: OffsetDateTime::now_utc(),
1417            },
1418        ))
1419        .await;
1420        let _ = pending.tx.send(resolution);
1421        Ok(true)
1422    }
1423
1424    /// Drops every pending external tool call for the turn and reports them as cancelled so an
1425    /// interrupt does not leak requests waiting on `tools/resolve`.
1426    async fn cancel_pending_external_tool_calls_for_turn(&self, turn_id: &TurnId) {
1427        let cancelled = {
1428            let mut pending = self.pending_external_tool_calls.lock().await;
1429            let request_ids = pending
1430                .iter()
1431                .filter(|(_, call)| &call.turn_id == turn_id)
1432                .map(|(request_id, _)| request_id.clone())
1433                .collect::<Vec<_>>();
1434            request_ids
1435                .into_iter()
1436                .filter_map(|request_id| pending.remove(&request_id).map(|call| (request_id, call)))
1437                .collect::<Vec<_>>()
1438        };
1439        for (request_id, call) in cancelled {
1440            self.emit(RoderEvent::ExternalToolCallResolved(
1441                ExternalToolCallResolved {
1442                    thread_id: call.thread_id,
1443                    turn_id: call.turn_id,
1444                    request_id,
1445                    tool_id: call.tool_id,
1446                    tool_name: call.tool_name,
1447                    outcome: ExternalToolCallOutcome::Cancelled,
1448                    is_error: true,
1449                    timestamp: OffsetDateTime::now_utc(),
1450                },
1451            ))
1452            .await;
1453        }
1454    }
1455
1456    pub async fn resolve_user_input(
1457        &self,
1458        request_id: &str,
1459        answers: serde_json::Value,
1460    ) -> anyhow::Result<bool> {
1461        let pending = self.pending_user_inputs.lock().await.remove(request_id);
1462        let Some(pending) = pending else {
1463            return Ok(false);
1464        };
1465        self.emit(RoderEvent::UserInputResolved(UserInputResolved {
1466            thread_id: pending.thread_id,
1467            turn_id: pending.turn_id,
1468            request_id: request_id.to_string(),
1469            answers: answers.clone(),
1470            timestamp: OffsetDateTime::now_utc(),
1471        }))
1472        .await;
1473        let _ = pending.tx.send(answers);
1474        Ok(true)
1475    }
1476
1477    async fn emit_plan_exit_resolved(
1478        &self,
1479        current: &PendingPlanExit,
1480        approved: bool,
1481        resolved_mode: PolicyMode,
1482    ) {
1483        self.emit(RoderEvent::PolicyExitPlanResolved(PolicyExitPlanResolved {
1484            thread_id: current.thread_id.clone(),
1485            turn_id: current.turn_id.clone(),
1486            request_id: current.request_id.clone(),
1487            approved,
1488            target_mode: current.target_mode,
1489            resolved_mode,
1490            timestamp: OffsetDateTime::now_utc(),
1491        }))
1492        .await;
1493    }
1494
1495    pub async fn select_provider(
1496        &self,
1497        provider: String,
1498        model: Option<String>,
1499        reasoning: Option<String>,
1500    ) -> anyhow::Result<RuntimeConfig> {
1501        let next = self
1502            .preview_provider_selection(provider, model, reasoning)
1503            .await?;
1504        let mut cfg = self.config.write().await;
1505        *cfg = next;
1506        Ok(cfg.clone())
1507    }
1508
1509    pub async fn preview_provider_selection(
1510        &self,
1511        provider: String,
1512        model: Option<String>,
1513        reasoning: Option<String>,
1514    ) -> anyhow::Result<RuntimeConfig> {
1515        self.engine_for(&provider)?;
1516        let mut cfg = self.config.read().await.clone();
1517        cfg.default_provider = provider;
1518        if let Some(model) = model {
1519            cfg.default_model = model;
1520        }
1521        if let Some(reasoning) = reasoning {
1522            if reasoning == REASONING_NONE
1523                && !model_supports_reasoning(&cfg.default_model, &reasoning)
1524            {
1525                return Ok(cfg.clone());
1526            }
1527            validate_reasoning_effort(&cfg.default_model, &reasoning)?;
1528            cfg.reasoning = Some(reasoning);
1529        }
1530        Ok(cfg)
1531    }
1532
1533    pub async fn effective_reasoning(&self) -> String {
1534        let cfg = self.config.read().await;
1535        effective_reasoning_for_model(&cfg, &cfg.default_model)
1536    }
1537
1538    pub fn effective_reasoning_for_config(cfg: &RuntimeConfig) -> String {
1539        effective_reasoning_for_model(cfg, &cfg.default_model)
1540    }
1541
1542    pub async fn set_dynamic_workflow_effort(
1543        &self,
1544        effort_profile: DynamicWorkflowEffortProfile,
1545    ) -> RuntimeConfig {
1546        let mut cfg = self.config.write().await;
1547        cfg.dynamic_workflows.effort_profile = effort_profile;
1548        cfg.clone()
1549    }
1550
1551    pub async fn dynamic_workflow_trigger_decision(
1552        &self,
1553        message: &str,
1554    ) -> WorkflowTriggerDecision {
1555        let cfg = self.config.read().await;
1556        classify_workflow_trigger(message, &cfg.dynamic_workflows)
1557    }
1558
1559    pub async fn create_thread(&self, title: Option<String>) -> anyhow::Result<ThreadMetadata> {
1560        self.create_thread_with(CreateThreadRequest {
1561            title,
1562            workspace: self.workspace.display().to_string(),
1563            workspace_id: None,
1564            root_id: None,
1565            provider: None,
1566            model: None,
1567            selection_mode: None,
1568            tool_allowlist: Vec::new(),
1569            developer_instructions: None,
1570            external_tools: Vec::new(),
1571            runner: None,
1572        })
1573        .await
1574    }
1575
1576    /**
1577     * Resolves an explicit thread-runner selection into a persisted binding.
1578     * Fails fast at thread creation when the provider is missing or rejects
1579     * the destination, instead of surfacing the error on the first tool call.
1580     */
1581    async fn resolve_thread_runner_binding(
1582        &self,
1583        thread_id: &str,
1584        selection: ThreadRunnerSelection,
1585    ) -> anyhow::Result<ThreadRunnerBinding> {
1586        let provider = self
1587            .registry
1588            .remote_runner_providers
1589            .iter()
1590            .find(|provider| provider.id() == selection.provider_id)
1591            .cloned()
1592            .ok_or_else(|| {
1593                anyhow::anyhow!(
1594                    "remote runner provider {:?} is not installed",
1595                    selection.provider_id
1596                )
1597            })?;
1598        let workspace = selection.workspace.trim();
1599        anyhow::ensure!(
1600            std::path::Path::new(workspace).is_absolute(),
1601            "runner workspace must be an absolute path on the runner: {workspace:?}"
1602        );
1603        let mut read_roots = Vec::with_capacity(selection.read_roots.len());
1604        for read_root in &selection.read_roots {
1605            let trimmed = read_root.trim();
1606            anyhow::ensure!(
1607                std::path::Path::new(trimmed).is_absolute(),
1608                "runner read root must be an absolute path on the runner: {trimmed:?}"
1609            );
1610            read_roots.push(PathBuf::from(trimmed));
1611        }
1612        let destination = RunnerDestination {
1613            id: format!("thread-{thread_id}"),
1614            provider_id: selection.provider_id,
1615            config: selection.config,
1616            default_manifest: roder_api::remote_runner::RunnerManifest::default(),
1617        };
1618        provider.validate_destination(&destination).await?;
1619        Ok(ThreadRunnerBinding {
1620            destination,
1621            workspace: PathBuf::from(workspace),
1622            read_roots,
1623        })
1624    }
1625
1626    /**
1627     * Build a per-thread binding from a runtime-level destination (selected via
1628     * the TUI runner picker or config `default_destination`) when the
1629     * destination's provider advertises a default workspace. Returns `None` when
1630     * there is no destination or the provider opts out (no default workspace),
1631     * preserving the legacy behavior where such threads keep local tools.
1632     */
1633    async fn synthesize_runtime_runner_binding(
1634        &self,
1635        thread_id: &str,
1636        destination: Option<RunnerDestination>,
1637    ) -> anyhow::Result<Option<ThreadRunnerBinding>> {
1638        let Some(destination) = destination else {
1639            return Ok(None);
1640        };
1641        let Some(provider) = self
1642            .registry
1643            .remote_runner_providers
1644            .iter()
1645            .find(|provider| provider.id() == destination.provider_id)
1646        else {
1647            return Ok(None);
1648        };
1649        // An explicit workspace in the destination config wins over the
1650        // provider default; absence of both means the provider opts out.
1651        let workspace = destination
1652            .config
1653            .get("working_dir")
1654            .and_then(serde_json::Value::as_str)
1655            .map(str::to_string)
1656            .or_else(|| provider.default_workspace());
1657        let Some(workspace) = workspace else {
1658            return Ok(None);
1659        };
1660        let selection = ThreadRunnerSelection {
1661            provider_id: destination.provider_id.clone(),
1662            config: destination.config.clone(),
1663            workspace,
1664            read_roots: Vec::new(),
1665        };
1666        Ok(Some(
1667            self.resolve_thread_runner_binding(thread_id, selection)
1668                .await?,
1669        ))
1670    }
1671
1672    /// Validates a runner selection without creating a thread; the placeholder destination id only appears in error messages.
1673    pub async fn validate_thread_runner_selection(
1674        &self,
1675        selection: ThreadRunnerSelection,
1676    ) -> anyhow::Result<()> {
1677        self.resolve_thread_runner_binding("validate", selection)
1678            .await
1679            .map(|_| ())
1680    }
1681
1682    pub async fn create_thread_with(
1683        &self,
1684        req: CreateThreadRequest,
1685    ) -> anyhow::Result<ThreadMetadata> {
1686        let cfg = self.config.read().await.clone();
1687        let now = OffsetDateTime::now_utc();
1688        let workspace = validate_thread_workspace(&req.workspace)?;
1689        let provider = req.provider.unwrap_or(cfg.default_provider);
1690        let model = req.model.unwrap_or(cfg.default_model);
1691        let selection_mode = req
1692            .selection_mode
1693            .unwrap_or_else(|| ModelSelectionMode::manual(provider.clone(), model.clone(), None));
1694        let thread_id = uuid::Uuid::new_v4().to_string();
1695        let runner_binding = match req.runner {
1696            Some(selection) => Some(
1697                self.resolve_thread_runner_binding(&thread_id, selection)
1698                    .await?,
1699            ),
1700            // No explicit per-thread selection: if a runtime-level destination
1701            // is active and its provider advertises a default workspace, bind
1702            // the new thread so its coding tools route into the runner. This is
1703            // what makes a TUI/config-selected runner (e.g. Blaxel) actually
1704            // execute tools in the sandbox instead of locally. Providers that
1705            // return no default workspace stay unbound (legacy local-tools).
1706            None => {
1707                self.synthesize_runtime_runner_binding(
1708                    &thread_id,
1709                    cfg.remote_runner_destination.clone(),
1710                )
1711                .await?
1712            }
1713        };
1714        let runner_destination = runner_binding
1715            .as_ref()
1716            .map(|binding| binding.destination.clone())
1717            .or_else(|| cfg.remote_runner_destination.clone());
1718        let metadata = ThreadMetadata {
1719            thread_id,
1720            title: req.title,
1721            workspace,
1722            workspace_id: req.workspace_id,
1723            root_id: req.root_id,
1724            provider: Some(provider),
1725            model: Some(model),
1726            selection_mode: Some(selection_mode),
1727            tool_allowlist: req.tool_allowlist,
1728            developer_instructions: req.developer_instructions,
1729            external_tools: req.external_tools,
1730            runner_destination,
1731            runner_state: None,
1732            runner_binding,
1733            created_at: now,
1734            updated_at: now,
1735            message_count: 0,
1736            usage: None,
1737            parent_thread_id: None,
1738            forked_from_turn_id: None,
1739            workspace_fork: None,
1740        };
1741
1742        let metadata = if let Some(store) = &self.thread_store {
1743            store.create_thread(metadata).await?
1744        } else {
1745            metadata
1746        };
1747        self.emit(RoderEvent::ThreadCreated(ThreadCreated {
1748            thread_id: metadata.thread_id.clone(),
1749            timestamp: OffsetDateTime::now_utc(),
1750        }))
1751        .await;
1752        Ok(metadata)
1753    }
1754
1755    pub async fn list_threads(&self) -> anyhow::Result<Vec<ThreadMetadata>> {
1756        if let Some(store) = &self.thread_store {
1757            return store.list_threads().await;
1758        }
1759        Ok(Vec::new())
1760    }
1761
1762    pub async fn list_threads_page(
1763        &self,
1764        options: roder_api::thread::ThreadListOptions,
1765    ) -> anyhow::Result<roder_api::thread::ThreadListPage> {
1766        if let Some(store) = &self.thread_store {
1767            return store.list_threads_page(options).await;
1768        }
1769        Ok(roder_api::thread::ThreadListPage::default())
1770    }
1771
1772    pub async fn load_thread_metadata(
1773        &self,
1774        thread_id: &str,
1775    ) -> anyhow::Result<Option<roder_api::thread::ThreadMetadata>> {
1776        if let Some(store) = &self.thread_store {
1777            return store.load_thread_metadata(&thread_id.to_string()).await;
1778        }
1779        Ok(None)
1780    }
1781
1782    pub async fn archive_thread(&self, thread_id: &str) -> anyhow::Result<bool> {
1783        let archived = if let Some(store) = &self.thread_store {
1784            store.archive_thread(&thread_id.to_string()).await?
1785        } else {
1786            false
1787        };
1788        if archived {
1789            self.thread_item_cache
1790                .lock()
1791                .await
1792                .remove_thread(&thread_id.to_string());
1793            self.evict_runner_session(&thread_id.to_string()).await;
1794        }
1795        Ok(archived)
1796    }
1797
1798    pub async fn start_team(&self, req: TeamStartRequest) -> anyhow::Result<TeamState> {
1799        let cfg = self.config.read().await.clone();
1800        let workspace = self.workspace.display().to_string();
1801        let lead_thread_id = match req.lead_thread_id {
1802            Some(thread_id) => thread_id,
1803            None => {
1804                self.create_thread_with(CreateThreadRequest {
1805                    title: Some("Team lead".to_string()),
1806                    workspace: workspace.clone(),
1807                    workspace_id: None,
1808                    root_id: None,
1809                    provider: None,
1810                    model: None,
1811                    selection_mode: None,
1812                    tool_allowlist: Vec::new(),
1813                    developer_instructions: None,
1814                    external_tools: Vec::new(),
1815                    runner: None,
1816                })
1817                .await?
1818                .thread_id
1819            }
1820        };
1821        let team_id = uuid::Uuid::new_v4().to_string();
1822        let active_lead_turn_id = self.active_turn_for_thread(&lead_thread_id).await;
1823        let lead_selection = match active_lead_turn_id.as_ref() {
1824            Some(turn_id) => self
1825                .active_turn_selections
1826                .read()
1827                .await
1828                .get(turn_id)
1829                .cloned(),
1830            None => self.selection_mode_for_thread(&lead_thread_id).await?,
1831        };
1832        let lead_concrete_selection = lead_selection
1833            .as_ref()
1834            .map(ModelSelectionMode::concrete_selection);
1835        let mut lead = crate::teams::lead_member(
1836            lead_thread_id.clone(),
1837            lead_concrete_selection
1838                .as_ref()
1839                .map(|selection| selection.provider.clone())
1840                .or_else(|| Some(cfg.default_provider.clone())),
1841            lead_concrete_selection
1842                .as_ref()
1843                .map(|selection| selection.model.clone())
1844                .or_else(|| Some(cfg.default_model.clone())),
1845            cfg.policy_mode,
1846        );
1847        if let Some(turn_id) = active_lead_turn_id {
1848            lead.current_turn_id = Some(turn_id);
1849            lead.status = TeamMemberStatus::Running;
1850        }
1851        let mut members = vec![lead];
1852
1853        for (index, member) in req.members.into_iter().enumerate() {
1854            let thread = self
1855                .create_thread_with(CreateThreadRequest {
1856                    title: Some(member.name.clone()),
1857                    workspace: workspace.clone(),
1858                    workspace_id: None,
1859                    root_id: None,
1860                    provider: member.model_provider.clone(),
1861                    model: member.model.clone(),
1862                    selection_mode: None,
1863                    tool_allowlist: Vec::new(),
1864                    developer_instructions: None,
1865                    external_tools: Vec::new(),
1866                    runner: None,
1867                })
1868                .await?;
1869            let member_id = format!("member-{}", index + 1);
1870            let descriptor = crate::teams::teammate_member(
1871                member_id.clone(),
1872                member.name,
1873                thread.thread_id.clone(),
1874                member.model_provider.or(thread.provider),
1875                member.model.or(thread.model),
1876                cfg.policy_mode,
1877            );
1878            members.push(descriptor);
1879        }
1880
1881        let now = OffsetDateTime::now_utc();
1882        let team = self
1883            .teams
1884            .insert(TeamState {
1885                id: team_id.clone(),
1886                lead_thread_id: lead_thread_id.clone(),
1887                display_mode: req.display_mode,
1888                members,
1889                mailbox: Vec::new(),
1890                tasks: Vec::new(),
1891                created_at: now,
1892                updated_at: now,
1893            })
1894            .await?;
1895        self.emit(RoderEvent::TeamStarted(TeamStarted {
1896            team_id: team_id.clone(),
1897            lead_thread_id,
1898            display_mode: team.display_mode,
1899            members: team.members.clone(),
1900            tasks: team.tasks.clone(),
1901            timestamp: OffsetDateTime::now_utc(),
1902        }))
1903        .await;
1904        for member in team
1905            .members
1906            .iter()
1907            .filter(|member| member.role != roder_api::teams::TeamMemberRole::Lead)
1908        {
1909            self.emit(RoderEvent::TeamMemberStarted(TeamMemberStarted {
1910                team_id: team_id.clone(),
1911                member: member.clone(),
1912                timestamp: OffsetDateTime::now_utc(),
1913            }))
1914            .await;
1915        }
1916        Ok(team)
1917    }
1918
1919    pub async fn list_teams(&self) -> Vec<TeamState> {
1920        let active_thread_ids = self
1921            .active_turns
1922            .read()
1923            .await
1924            .values()
1925            .map(|handle| handle.thread_id.clone())
1926            .collect::<std::collections::HashSet<_>>();
1927        self.teams
1928            .list()
1929            .await
1930            .into_iter()
1931            .map(|team| runtime_local_team_view(team, &active_thread_ids))
1932            .collect()
1933    }
1934
1935    pub async fn read_team(&self, team_id: &str) -> Option<TeamState> {
1936        let active_thread_ids = self
1937            .active_turns
1938            .read()
1939            .await
1940            .values()
1941            .map(|handle| handle.thread_id.clone())
1942            .collect::<std::collections::HashSet<_>>();
1943        self.teams
1944            .get(team_id)
1945            .await
1946            .map(|team| runtime_local_team_view(team, &active_thread_ids))
1947    }
1948
1949    pub async fn start_team_member(
1950        &self,
1951        team_id: &str,
1952        req: TeamMemberStartRequest,
1953    ) -> anyhow::Result<TeamState> {
1954        self.start_team_member_with_selection(team_id, req, None)
1955            .await
1956    }
1957
1958    pub async fn message_team_member(
1959        self: &Arc<Self>,
1960        team_id: &str,
1961        member_id: &str,
1962        message: String,
1963    ) -> anyhow::Result<TurnId> {
1964        let team = self
1965            .read_team(team_id)
1966            .await
1967            .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
1968        self.followup_team_member(&team.lead_thread_id, team_id, member_id, message)
1969            .await
1970    }
1971
1972    /// Queue a mailbox message without starting an idle agent. If the target is
1973    /// already running, the message is delivered at the next inference boundary.
1974    pub(crate) async fn queue_team_member_message(
1975        self: &Arc<Self>,
1976        caller_thread_id: &ThreadId,
1977        team_id: &str,
1978        member_id: &str,
1979        message: String,
1980    ) -> anyhow::Result<Option<TurnId>> {
1981        let _delivery_guard = self.agent_team_spawn_lock.lock().await;
1982        let team = self
1983            .read_team(team_id)
1984            .await
1985            .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
1986        let member = team
1987            .members
1988            .iter()
1989            .find(|member| member.id == member_id)
1990            .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
1991            .clone();
1992        if member.status == TeamMemberStatus::Closed {
1993            anyhow::bail!("subagent {} is closed", member.name);
1994        }
1995        let from_member_id = team
1996            .members
1997            .iter()
1998            .find(|candidate| candidate.thread_id == *caller_thread_id)
1999            .map(|candidate| candidate.id.clone());
2000        self.teams
2001            .append_mailbox_message(
2002                team_id,
2003                from_member_id,
2004                member_id.to_string(),
2005                TeamMailboxMessageKind::Message,
2006                message,
2007            )
2008            .await?;
2009        let Some(turn_id) = self.active_turn_for_thread(&member.thread_id).await else {
2010            return Ok(None);
2011        };
2012        self.deliver_pending_team_mailbox(team_id, member_id, member.thread_id, turn_id.clone())
2013            .await?;
2014        Ok(Some(turn_id))
2015    }
2016
2017    /// Deliver queued messages and start an idle agent, or steer its active turn.
2018    pub(crate) async fn followup_team_member(
2019        self: &Arc<Self>,
2020        caller_thread_id: &ThreadId,
2021        team_id: &str,
2022        member_id: &str,
2023        message: String,
2024    ) -> anyhow::Result<TurnId> {
2025        let _spawn_guard = self.agent_team_spawn_lock.lock().await;
2026        self.followup_team_member_locked(caller_thread_id, team_id, member_id, message)
2027            .await
2028    }
2029
2030    async fn followup_team_member_locked(
2031        self: &Arc<Self>,
2032        caller_thread_id: &ThreadId,
2033        team_id: &str,
2034        member_id: &str,
2035        message: String,
2036    ) -> anyhow::Result<TurnId> {
2037        let team = self
2038            .read_team(team_id)
2039            .await
2040            .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2041        let member = team
2042            .members
2043            .iter()
2044            .find(|member| member.id == member_id)
2045            .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2046            .clone();
2047        if member.status == TeamMemberStatus::Closed {
2048            anyhow::bail!("subagent {} is closed", member.name);
2049        }
2050        let from_member_id = team
2051            .members
2052            .iter()
2053            .find(|candidate| candidate.thread_id == *caller_thread_id)
2054            .map(|candidate| candidate.id.clone());
2055        self.teams
2056            .append_mailbox_message(
2057                team_id,
2058                from_member_id,
2059                member_id.to_string(),
2060                TeamMailboxMessageKind::NewTask,
2061                message,
2062            )
2063            .await?;
2064        if let Some(turn_id) = self.active_turn_for_thread(&member.thread_id).await {
2065            self.deliver_pending_team_mailbox(
2066                team_id,
2067                member_id,
2068                member.thread_id,
2069                turn_id.clone(),
2070            )
2071            .await?;
2072            return Ok(turn_id);
2073        }
2074
2075        self.ensure_codex_v2_team_capacity(&team, &member.id, None)
2076            .await?;
2077        let metadata = self.load_thread_metadata(&member.thread_id).await?;
2078        let inherited_context = self
2079            .team_member_turn_contexts
2080            .lock()
2081            .await
2082            .get(&member.thread_id)
2083            .cloned();
2084        let workspace = inherited_context
2085            .as_ref()
2086            .map(|context| context.workspace.clone())
2087            .or_else(|| metadata.as_ref().map(|metadata| metadata.workspace.clone()))
2088            .unwrap_or_else(|| self.workspace.display().to_string());
2089        let member_thread_id = member.thread_id;
2090        let turn_id = self
2091            .start_turn(StartTurnRequest {
2092                thread_id: member_thread_id.clone(),
2093                message: String::new(),
2094                images: Vec::new(),
2095                provider_override: member.model_provider,
2096                model_override: member.model,
2097                reasoning_override: metadata
2098                    .as_ref()
2099                    .and_then(|metadata| metadata.selection_mode.as_ref())
2100                    .and_then(ModelSelectionMode::reasoning)
2101                    .map(str::to_string),
2102                workspace,
2103                instructions: inherited_context
2104                    .as_ref()
2105                    .map(|context| context.instructions.clone())
2106                    .unwrap_or_else(crate::default_instructions),
2107                developer_context: inherited_context
2108                    .as_ref()
2109                    .and_then(|context| context.developer_context.clone()),
2110                task_ledger_required: false,
2111            })
2112            .await?;
2113        Ok(turn_id)
2114    }
2115
2116    pub async fn set_team_member_policy_mode(
2117        &self,
2118        team_id: &str,
2119        member_id: &str,
2120        policy_mode: PolicyMode,
2121    ) -> anyhow::Result<TeamState> {
2122        self.teams
2123            .set_member_policy_mode(team_id, member_id, policy_mode)
2124            .await
2125    }
2126
2127    pub async fn interrupt_team_member(
2128        &self,
2129        team_id: &str,
2130        member_id: &str,
2131    ) -> anyhow::Result<Option<TurnId>> {
2132        let _interrupt_guard = self.agent_team_spawn_lock.lock().await;
2133        let team = self
2134            .read_team(team_id)
2135            .await
2136            .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2137        let member = team
2138            .members
2139            .iter()
2140            .find(|member| member.id == member_id)
2141            .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2142            .clone();
2143        if member.status != TeamMemberStatus::Running {
2144            return Ok(None);
2145        }
2146        let Some(turn_id) = member.current_turn_id.clone() else {
2147            return Ok(None);
2148        };
2149        if self
2150            .active_turn_for_thread(&member.thread_id)
2151            .await
2152            .as_ref()
2153            != Some(&turn_id)
2154        {
2155            return Ok(None);
2156        }
2157        self.interrupt_turn(member.thread_id.clone(), turn_id.clone())
2158            .await?;
2159        // Make queued mailbox work immediately eligible for the replacement turn while the
2160        // lifecycle lock still prevents a follow-up from starting. Delivery acknowledgements
2161        // are turn-owned, so a late acknowledgement from the aborted turn cannot steal a
2162        // reservation acquired by its replacement.
2163        self.teams
2164            .release_mailbox_reservations_for_turn(&turn_id)
2165            .await;
2166        self.teams
2167            .update_member(team_id, member_id, |member| {
2168                member.status = TeamMemberStatus::Interrupted;
2169                member.current_turn_id = None;
2170                member.terminal_error = None;
2171            })
2172            .await?;
2173        self.emit(RoderEvent::TeamMemberCompleted(TeamMemberCompleted {
2174            team_id: team_id.to_string(),
2175            member_id: member_id.to_string(),
2176            member_thread_id: member.thread_id,
2177            turn_id: Some(turn_id.clone()),
2178            status: TeamMemberStatus::Interrupted,
2179            final_message: member.final_message,
2180            error: None,
2181            timestamp: OffsetDateTime::now_utc(),
2182        }))
2183        .await;
2184        Ok(Some(turn_id))
2185    }
2186
2187    pub async fn close_team_member(
2188        &self,
2189        team_id: &str,
2190        member_id: &str,
2191    ) -> anyhow::Result<roder_api::teams::TeamMemberDescriptor> {
2192        let team = self
2193            .read_team(team_id)
2194            .await
2195            .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2196        let member = team
2197            .members
2198            .iter()
2199            .find(|member| member.id == member_id)
2200            .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2201            .clone();
2202        if member.role == roder_api::teams::TeamMemberRole::Lead {
2203            anyhow::bail!("team lead cannot be closed as a subagent");
2204        }
2205        let interrupted_turn_id = if member.status == TeamMemberStatus::Running {
2206            if let Some(turn_id) = member.current_turn_id.clone() {
2207                self.interrupt_turn(member.thread_id.clone(), turn_id.clone())
2208                    .await?;
2209                Some(turn_id)
2210            } else {
2211                None
2212            }
2213        } else {
2214            member.current_turn_id.clone()
2215        };
2216        let updated = self
2217            .teams
2218            .update_member(team_id, member_id, |member| {
2219                member.status = TeamMemberStatus::Closed;
2220                member.current_turn_id = None;
2221            })
2222            .await?;
2223        let closed = updated
2224            .members
2225            .iter()
2226            .find(|member| member.id == member_id)
2227            .cloned()
2228            .ok_or_else(|| anyhow::anyhow!("closed team member disappeared"))?;
2229        self.team_member_turn_contexts
2230            .lock()
2231            .await
2232            .remove(&closed.thread_id);
2233        self.emit(RoderEvent::TeamMemberCompleted(TeamMemberCompleted {
2234            team_id: team_id.to_string(),
2235            member_id: closed.id.clone(),
2236            member_thread_id: closed.thread_id.clone(),
2237            turn_id: interrupted_turn_id,
2238            status: TeamMemberStatus::Closed,
2239            final_message: closed.final_message.clone(),
2240            error: closed.terminal_error.clone(),
2241            timestamp: OffsetDateTime::now_utc(),
2242        }))
2243        .await;
2244        Ok(closed)
2245    }
2246
2247    pub async fn cleanup_team(&self, team_id: &str, force: bool) -> anyhow::Result<bool> {
2248        let Some(team) = self.read_team(team_id).await else {
2249            return Ok(false);
2250        };
2251        if !force
2252            && team
2253                .members
2254                .iter()
2255                .any(|member| member.status == TeamMemberStatus::Running)
2256        {
2257            anyhow::bail!("team {team_id:?} has active teammates; use forced cleanup");
2258        }
2259        if force {
2260            for member in team.members.iter().filter(|member| {
2261                member.role != roder_api::teams::TeamMemberRole::Lead
2262                    && member.status == TeamMemberStatus::Running
2263            }) {
2264                if let Some(turn_id) = member
2265                    .current_turn_id
2266                    .clone()
2267                    .or(self.active_turn_for_thread(&member.thread_id).await)
2268                {
2269                    let _ = self.interrupt_turn(member.thread_id.clone(), turn_id).await;
2270                }
2271            }
2272        }
2273        let removed = self.teams.remove(team_id).await?.is_some();
2274        if removed {
2275            let member_thread_ids = team
2276                .members
2277                .iter()
2278                .map(|member| member.thread_id.clone())
2279                .collect::<std::collections::HashSet<_>>();
2280            self.team_member_turn_contexts
2281                .lock()
2282                .await
2283                .retain(|thread_id, _| !member_thread_ids.contains(thread_id));
2284            self.emit(RoderEvent::TeamCleanupCompleted(TeamCleanupCompleted {
2285                team_id: team_id.to_string(),
2286                forced: force,
2287                timestamp: OffsetDateTime::now_utc(),
2288            }))
2289            .await;
2290        }
2291        Ok(removed)
2292    }
2293
2294    pub async fn effective_policy_mode_for_thread(&self, thread_id: &str) -> PolicyMode {
2295        if let Some(mode) = self.teams.policy_mode_for_thread(thread_id).await {
2296            return mode;
2297        }
2298        self.status().await.policy_mode
2299    }
2300
2301    async fn complete_team_member_turn_with_result(
2302        &self,
2303        thread_id: &ThreadId,
2304        turn_id: &TurnId,
2305        status: TeamMemberStatus,
2306        final_message: Option<String>,
2307        terminal_error: Option<String>,
2308    ) -> anyhow::Result<()> {
2309        let _delivery_guard = self.agent_team_spawn_lock.lock().await;
2310        let Some((team_id, member)) = self
2311            .teams
2312            .complete_member_turn(
2313                thread_id,
2314                turn_id,
2315                status,
2316                final_message.clone(),
2317                terminal_error.clone(),
2318            )
2319            .await?
2320        else {
2321            return Ok(());
2322        };
2323        if let Some(parent_thread_id) = member.parent_thread_id.as_ref()
2324            && let Some(team) = self.read_team(&team_id).await
2325            && let Some(parent) = team
2326                .members
2327                .iter()
2328                .find(|candidate| candidate.thread_id == *parent_thread_id)
2329        {
2330            let identity = member
2331                .agent_path
2332                .as_deref()
2333                .or(member.task_name.as_deref())
2334                .unwrap_or(&member.name);
2335            let mut report = format!("Agent {identity} finished with status {status:?}.");
2336            if let Some(message) = final_message.as_deref()
2337                && !message.trim().is_empty()
2338            {
2339                report.push_str("\n\nFinal result:\n");
2340                report.push_str(message);
2341            }
2342            if let Some(error) = terminal_error.as_deref()
2343                && !error.trim().is_empty()
2344            {
2345                report.push_str("\n\nTerminal error:\n");
2346                report.push_str(error);
2347            }
2348            let mailbox_appended = self
2349                .teams
2350                .append_mailbox_message(
2351                    &team_id,
2352                    Some(member.id.clone()),
2353                    parent.id.clone(),
2354                    TeamMailboxMessageKind::FinalAnswer,
2355                    report,
2356                )
2357                .await
2358                .is_ok();
2359            if mailbox_appended
2360                && let Some(parent_turn_id) = self.active_turn_for_thread(parent_thread_id).await
2361            {
2362                // The parent may finish between the active-turn lookup and enqueue. Its durable
2363                // mailbox entry remains pending for the next turn, while terminal observers must
2364                // still receive TeamMemberCompleted for this child.
2365                let _ = self
2366                    .deliver_pending_team_mailbox(
2367                        &team_id,
2368                        &parent.id,
2369                        parent_thread_id.clone(),
2370                        parent_turn_id,
2371                    )
2372                    .await;
2373            }
2374        }
2375        self.emit(RoderEvent::TeamMemberCompleted(TeamMemberCompleted {
2376            team_id,
2377            member_id: member.id,
2378            member_thread_id: member.thread_id,
2379            turn_id: Some(turn_id.clone()),
2380            status,
2381            final_message,
2382            error: terminal_error,
2383            timestamp: OffsetDateTime::now_utc(),
2384        }))
2385        .await;
2386        Ok(())
2387    }
2388
2389    /// Stops accepting new turns, requests interruption for every active turn,
2390    /// and waits for their runtime tasks to reach terminal cleanup up to
2391    /// `timeout`. Repeated calls are safe and continue draining the same set.
2392    pub async fn drain_active_turns(&self, timeout: std::time::Duration) -> RuntimeDrainOutcome {
2393        let started_at = tokio::time::Instant::now();
2394        let persistence_failures_before =
2395            self.lifecycle_persistence_failures.load(Ordering::Acquire);
2396        let turn_admission = self.turn_admission.lock().await;
2397        self.accepting_turns.store(false, Ordering::Release);
2398        let active = self
2399            .active_turns
2400            .read()
2401            .await
2402            .iter()
2403            .map(|(turn_id, handle)| (turn_id.clone(), handle.thread_id.clone()))
2404            .collect::<Vec<_>>();
2405        let draining = self
2406            .turn_drains
2407            .read()
2408            .await
2409            .iter()
2410            .map(|(turn_id, drain)| (turn_id.clone(), drain.thread_id.clone()))
2411            .collect::<Vec<_>>();
2412        drop(turn_admission);
2413
2414        let mut interrupted_turns = std::collections::BTreeMap::new();
2415        for (turn_id, thread_id) in active.iter().chain(draining.iter()) {
2416            interrupted_turns.insert(turn_id.clone(), thread_id.clone());
2417        }
2418        let interrupted_turn_ids = interrupted_turns.keys().cloned().collect::<Vec<_>>();
2419        for (turn_id, thread_id) in active {
2420            let _ = self
2421                .interrupt_turn_with_reason(thread_id, turn_id, TurnLifecycleReason::Shutdown)
2422                .await;
2423        }
2424
2425        let wait_for_empty = async {
2426            loop {
2427                let notified = self.active_turns_changed.notified();
2428                if self.active_turns.read().await.is_empty()
2429                    && self.turn_drains.read().await.is_empty()
2430                {
2431                    return;
2432                }
2433                notified.await;
2434            }
2435        };
2436        let outcome = if tokio::time::timeout(timeout, wait_for_empty).await.is_ok() {
2437            if self.lifecycle_persistence_failures.load(Ordering::Acquire)
2438                > persistence_failures_before
2439            {
2440                RuntimeDrainOutcome::PersistenceFailed {
2441                    interrupted_turn_ids,
2442                    remaining_turn_ids: Vec::new(),
2443                }
2444            } else {
2445                RuntimeDrainOutcome::Clean {
2446                    interrupted_turn_ids,
2447                }
2448            }
2449        } else {
2450            let active_remaining = self
2451                .active_turns
2452                .read()
2453                .await
2454                .iter()
2455                .map(|(turn_id, handle)| (turn_id.clone(), handle.drain.clone()))
2456                .collect::<Vec<_>>();
2457            let draining_remaining = self
2458                .turn_drains
2459                .read()
2460                .await
2461                .iter()
2462                .map(|(turn_id, drain)| (turn_id.clone(), drain.clone()))
2463                .collect::<Vec<_>>();
2464            let remaining = active_remaining
2465                .into_iter()
2466                .chain(draining_remaining)
2467                .collect::<std::collections::BTreeMap<_, _>>();
2468            let remaining_turn_ids = remaining.keys().cloned().collect::<Vec<_>>();
2469            for turn_id in &remaining_turn_ids {
2470                let handle = remaining
2471                    .get(turn_id)
2472                    .expect("remaining turn ID must have a drain handle");
2473                self.record_turn_lifecycle(
2474                    handle.thread_id.clone(),
2475                    turn_id.clone(),
2476                    TurnLifecycleState::InterruptRequested,
2477                    TurnCleanupState::TimedOut,
2478                    Some(TurnLifecycleReason::Shutdown),
2479                )
2480                .await;
2481            }
2482            if self.lifecycle_persistence_failures.load(Ordering::Acquire)
2483                > persistence_failures_before
2484            {
2485                RuntimeDrainOutcome::PersistenceFailed {
2486                    interrupted_turn_ids,
2487                    remaining_turn_ids,
2488                }
2489            } else {
2490                RuntimeDrainOutcome::DeadlineExceeded {
2491                    interrupted_turn_ids,
2492                    remaining_turn_ids,
2493                }
2494            }
2495        };
2496        self.record_lifecycle_drain_outcome(started_at, &outcome);
2497        outcome
2498    }
2499
2500    /// Enables turn submission after a prior drain attempt. This is intended
2501    /// for embedders that keep a runtime alive after a bounded drain timeout.
2502    pub fn resume_accepting_turns(&self) {
2503        self.accepting_turns.store(true, Ordering::Release);
2504    }
2505
2506    pub async fn turn_lifecycle_snapshot(
2507        &self,
2508        thread_id: &ThreadId,
2509    ) -> anyhow::Result<TurnLifecycleSnapshot> {
2510        let Some(store) = &self.thread_store else {
2511            return Ok(TurnLifecycleSnapshot::default());
2512        };
2513        let extension_states = store.load_extension_states(thread_id).await?;
2514        Ok(turn_lifecycle_snapshot(&extension_states))
2515    }
2516
2517    pub async fn load_thread(
2518        &self,
2519        thread_id: &ThreadId,
2520    ) -> anyhow::Result<Option<ThreadSnapshot>> {
2521        let loaded = if let Some(store) = &self.thread_store {
2522            store.load_thread(thread_id).await?
2523        } else {
2524            None
2525        };
2526        if let Some(snapshot) = loaded.as_ref() {
2527            let live_turn_ids = self
2528                .active_turns
2529                .read()
2530                .await
2531                .keys()
2532                .cloned()
2533                .chain(self.turn_drains.read().await.keys().cloned())
2534                .collect::<std::collections::HashSet<_>>();
2535            let lifecycle = turn_lifecycle_snapshot(&snapshot.extension_states);
2536            for record in lifecycle.records.into_iter().filter(|record| {
2537                record.state.requires_recovery() && !live_turn_ids.contains(&record.turn_id)
2538            }) {
2539                self.lifecycle_restart_reconciliations
2540                    .fetch_add(1, Ordering::AcqRel);
2541                self.record_turn_lifecycle(
2542                    record.thread_id,
2543                    record.turn_id,
2544                    TurnLifecycleState::RecoveryNeeded,
2545                    TurnCleanupState::Unknown,
2546                    Some(TurnLifecycleReason::RuntimeRestart),
2547                )
2548                .await;
2549            }
2550            self.emit(RoderEvent::ThreadLoaded(ThreadLoaded {
2551                thread_id: thread_id.clone(),
2552                timestamp: OffsetDateTime::now_utc(),
2553            }))
2554            .await;
2555        }
2556        Ok(loaded)
2557    }
2558
2559    pub async fn workspace_for_thread(&self, thread_id: &ThreadId) -> anyhow::Result<String> {
2560        if let Some(store) = &self.thread_store {
2561            let snapshot = store
2562                .load_thread(thread_id)
2563                .await?
2564                .ok_or_else(|| anyhow::anyhow!("thread not found: {thread_id}"));
2565            match snapshot {
2566                Ok(snapshot) => {
2567                    if let Some(metadata) = snapshot.metadata {
2568                        // Fork-backed threads fail closed before any write
2569                        // when their workspace was removed out-of-band.
2570                        if let Some(fork) = &metadata.workspace_fork
2571                            && fork.status == roder_api::forks::ForkStatus::Active
2572                            && !std::path::Path::new(&metadata.workspace).is_dir()
2573                        {
2574                            anyhow::bail!(
2575                                "workspace fork {} is missing its workspace at {}; restore it or \
2576                                 remove the fork before running turns in this thread",
2577                                fork.id,
2578                                metadata.workspace
2579                            );
2580                        }
2581                        return Ok(metadata.workspace);
2582                    }
2583                    eprintln!(
2584                        "thread metadata missing while resolving workspace for {thread_id}; falling back to runtime workspace"
2585                    );
2586                }
2587                Err(err) => {
2588                    eprintln!(
2589                        "thread missing while resolving workspace for {thread_id}: {err}; falling back to runtime workspace"
2590                    );
2591                }
2592            }
2593        }
2594        Ok(self.workspace.display().to_string())
2595    }
2596
2597    async fn selection_mode_for_thread(
2598        &self,
2599        thread_id: &ThreadId,
2600    ) -> anyhow::Result<Option<ModelSelectionMode>> {
2601        let Some(store) = &self.thread_store else {
2602            return Ok(None);
2603        };
2604        Ok(store
2605            .load_thread(thread_id)
2606            .await?
2607            .and_then(|snapshot| snapshot.metadata)
2608            .and_then(|metadata| {
2609                metadata
2610                    .selection_mode
2611                    .or_else(|| match (metadata.provider, metadata.model) {
2612                        (Some(provider), Some(model)) => {
2613                            Some(ModelSelectionMode::manual(provider, model, None))
2614                        }
2615                        _ => None,
2616                    })
2617            }))
2618    }
2619
2620    /// Per-thread tool allowlist, developer instructions, and external tools persisted at thread creation.
2621    pub(crate) async fn thread_turn_overrides(
2622        &self,
2623        thread_id: &ThreadId,
2624    ) -> anyhow::Result<ThreadTurnOverrides> {
2625        let Some(store) = &self.thread_store else {
2626            return Ok(ThreadTurnOverrides::default());
2627        };
2628        Ok(store
2629            .load_thread_metadata(thread_id)
2630            .await?
2631            .map(|metadata| ThreadTurnOverrides {
2632                tool_allowlist: metadata.tool_allowlist,
2633                developer_instructions: metadata.developer_instructions,
2634                external_tools: metadata.external_tools,
2635            })
2636            .unwrap_or_default())
2637    }
2638
2639    pub async fn set_thread_selection_mode(
2640        &self,
2641        thread_id: &ThreadId,
2642        selection_mode: ModelSelectionMode,
2643    ) -> anyhow::Result<()> {
2644        let Some(store) = &self.thread_store else {
2645            return Ok(());
2646        };
2647        let Some(snapshot) = store.load_thread(thread_id).await? else {
2648            anyhow::bail!("thread not found: {thread_id}");
2649        };
2650        let Some(mut metadata) = snapshot.metadata else {
2651            return Ok(());
2652        };
2653        let concrete = selection_mode.concrete_selection();
2654        metadata.provider = Some(concrete.provider);
2655        metadata.model = Some(concrete.model);
2656        metadata.selection_mode = Some(selection_mode);
2657        metadata.updated_at = OffsetDateTime::now_utc();
2658        store.update_thread_metadata(metadata).await?;
2659        Ok(())
2660    }
2661
2662    async fn runner_session_for_thread(
2663        &self,
2664        thread_id: &ThreadId,
2665    ) -> anyhow::Result<Option<(RunnerDestination, Arc<dyn RemoteRunnerSession>)>> {
2666        let metadata = if let Some(store) = &self.thread_store {
2667            store.load_thread_metadata(thread_id).await?
2668        } else {
2669            None
2670        };
2671        // An explicit per-thread binding wins over the runtime-level destination.
2672        let destination = metadata
2673            .as_ref()
2674            .and_then(|metadata| metadata.runner_binding.as_ref())
2675            .map(|binding| binding.destination.clone())
2676            .or(self.config.read().await.remote_runner_destination.clone());
2677        let Some(destination) = destination else {
2678            self.evict_runner_session(thread_id).await;
2679            return Ok(None);
2680        };
2681        let provider = self
2682            .registry
2683            .remote_runner_providers
2684            .iter()
2685            .find(|provider| provider.id() == destination.provider_id)
2686            .cloned()
2687            .ok_or_else(|| {
2688                anyhow::anyhow!(
2689                    "remote runner provider {:?} is not installed",
2690                    destination.provider_id
2691                )
2692            })?;
2693        let persisted_state = metadata.and_then(|metadata| metadata.runner_state);
2694        let slot = {
2695            let mut sessions = self.runner_sessions.lock().await;
2696            match sessions.get(thread_id) {
2697                Some(slot) if slot.matches(&destination) => slot.clone(),
2698                _ => {
2699                    let slot = Arc::new(RunnerSessionSlot::empty(&destination));
2700                    sessions.insert(thread_id.clone(), slot.clone());
2701                    slot
2702                }
2703            }
2704        };
2705
2706        loop {
2707            let initialized = slot.initialized.notified();
2708            let mut state = slot.state.lock().await;
2709            match &*state {
2710                RunnerSessionSlotState::Ready(cached) => {
2711                    return Ok(Some((cached.destination.clone(), cached.session.clone())));
2712                }
2713                RunnerSessionSlotState::Failed(error) => {
2714                    return Err(anyhow::anyhow!(error.to_string()));
2715                }
2716                RunnerSessionSlotState::Initializing => {
2717                    drop(state);
2718                    initialized.await;
2719                }
2720                RunnerSessionSlotState::Empty => {
2721                    *state = RunnerSessionSlotState::Initializing;
2722                    drop(state);
2723                    self.spawn_runner_session_initialization(
2724                        thread_id.clone(),
2725                        destination.clone(),
2726                        provider.clone(),
2727                        persisted_state.clone(),
2728                        slot.clone(),
2729                    );
2730                }
2731            }
2732        }
2733    }
2734
2735    fn spawn_runner_session_initialization(
2736        &self,
2737        thread_id: ThreadId,
2738        destination: RunnerDestination,
2739        provider: Arc<dyn RemoteRunnerProvider>,
2740        persisted_state: Option<RunnerSessionState>,
2741        slot: Arc<RunnerSessionSlot>,
2742    ) {
2743        let thread_store = self.thread_store.clone();
2744        let runner_sessions = self.runner_sessions.clone();
2745        // The task intentionally outlives the turn that first requested the
2746        // workspace. Interrupting that turn must not strand the slot in
2747        // `Initializing` and deadlock every later tool or lifecycle call.
2748        drop(tokio::spawn(async move {
2749            let initialized = async {
2750                let session = if let Some(state) = persisted_state
2751                    && state.provider_id == destination.provider_id
2752                    && state.destination_id == destination.id
2753                {
2754                    match provider.resume_session(state).await {
2755                        Ok(session) => session,
2756                        Err(_) => provider.create_session(destination.clone()).await?,
2757                    }
2758                } else {
2759                    provider.create_session(destination.clone()).await?
2760                };
2761                let resolved = (destination.clone(), session.clone());
2762                // Persist before releasing the singleflight or dispatching
2763                // the first tool. A later process can rejoin even if the turn
2764                // that requested initialization has already been interrupted.
2765                Self::persist_runner_state_in_store(
2766                    thread_store.as_ref(),
2767                    &thread_id,
2768                    Some(&resolved),
2769                )
2770                .await?;
2771                Ok::<_, anyhow::Error>(CachedRunnerSession {
2772                    destination,
2773                    session,
2774                })
2775            }
2776            .await;
2777
2778            match initialized {
2779                Ok(cached) => {
2780                    *slot.state.lock().await = RunnerSessionSlotState::Ready(cached);
2781                    slot.initialized.notify_waiters();
2782                }
2783                Err(error) => {
2784                    let message: Arc<str> = Arc::from(format!("{error:#}"));
2785                    *slot.state.lock().await = RunnerSessionSlotState::Failed(message);
2786                    slot.initialized.notify_waiters();
2787                    let mut sessions = runner_sessions.lock().await;
2788                    let is_current = sessions
2789                        .get(&thread_id)
2790                        .is_some_and(|current| Arc::ptr_eq(current, &slot));
2791                    if is_current {
2792                        sessions.remove(&thread_id);
2793                    }
2794                }
2795            }
2796        }));
2797    }
2798
2799    async fn evict_runner_session(&self, thread_id: &ThreadId) {
2800        self.runner_sessions.lock().await.remove(thread_id);
2801    }
2802
2803    async fn cache_runner_session(
2804        &self,
2805        thread_id: &ThreadId,
2806        destination: RunnerDestination,
2807        session: Arc<dyn RemoteRunnerSession>,
2808    ) {
2809        let cached = CachedRunnerSession {
2810            destination,
2811            session,
2812        };
2813        self.runner_sessions.lock().await.insert(
2814            thread_id.clone(),
2815            Arc::new(RunnerSessionSlot::ready(cached)),
2816        );
2817    }
2818
2819    /// Read a thread's explicit runner binding without creating or resuming a
2820    /// session. Tool routing uses this before local previews so runner-backed
2821    /// and fail-closed hosted calls never inspect the host workspace.
2822    pub(crate) async fn runner_binding_for_thread(
2823        &self,
2824        thread_id: &ThreadId,
2825    ) -> anyhow::Result<Option<ThreadRunnerBinding>> {
2826        let Some(store) = &self.thread_store else {
2827            return Ok(None);
2828        };
2829        Ok(store
2830            .load_thread_metadata(thread_id)
2831            .await?
2832            .and_then(|metadata| metadata.runner_binding))
2833    }
2834
2835    /// Resolve a previously-read binding into a live remote workspace. The
2836    /// session remains lazy: callers invoke this only after policy approval.
2837    pub(crate) async fn remote_workspace_for_binding(
2838        &self,
2839        thread_id: &ThreadId,
2840        binding: ThreadRunnerBinding,
2841    ) -> anyhow::Result<Arc<RemoteWorkspace>> {
2842        let session = self
2843            .runner_session_for_thread(thread_id)
2844            .await?
2845            .map(|(_, session)| session)
2846            .ok_or_else(|| {
2847                anyhow::anyhow!("runner-bound thread {thread_id} has no runner session")
2848            })?;
2849        Ok(Arc::new(RemoteWorkspace {
2850            session,
2851            root: binding.workspace,
2852            read_roots: binding.read_roots,
2853        }))
2854    }
2855
2856    pub(crate) async fn runner_tool_execution_lock(
2857        &self,
2858        session: &dyn RemoteRunnerSession,
2859    ) -> Arc<RunnerToolExecutionMutex> {
2860        let state = session.state();
2861        let key = (state.provider_id, state.session_id);
2862        let mut locks = self.runner_tool_execution_locks.lock().await;
2863        if let Some(lock) = locks.get(&key).and_then(Weak::upgrade) {
2864            return lock;
2865        }
2866
2867        locks.retain(|_, lock| lock.strong_count() > 0);
2868        let lock = Arc::new(Mutex::new(()));
2869        locks.insert(key, Arc::downgrade(&lock));
2870        lock
2871    }
2872
2873    async fn persist_runner_state(
2874        &self,
2875        thread_id: &ThreadId,
2876        runner: Option<&(RunnerDestination, Arc<dyn RemoteRunnerSession>)>,
2877    ) -> anyhow::Result<()> {
2878        Self::persist_runner_state_in_store(self.thread_store.as_ref(), thread_id, runner).await
2879    }
2880
2881    async fn persist_runner_state_in_store(
2882        store: Option<&Arc<dyn ThreadStore>>,
2883        thread_id: &ThreadId,
2884        runner: Option<&(RunnerDestination, Arc<dyn RemoteRunnerSession>)>,
2885    ) -> anyhow::Result<()> {
2886        let Some((destination, session)) = runner else {
2887            return Ok(());
2888        };
2889        let Some(store) = store else {
2890            return Ok(());
2891        };
2892        let Some(snapshot) = store.load_thread(thread_id).await? else {
2893            return Ok(());
2894        };
2895        let Some(mut metadata) = snapshot.metadata else {
2896            return Ok(());
2897        };
2898        metadata.runner_destination = Some(destination.clone());
2899        metadata.runner_state = Some(session.state());
2900        metadata.updated_at = OffsetDateTime::now_utc();
2901        store.update_thread_metadata(metadata).await?;
2902        Ok(())
2903    }
2904
2905    fn remote_runner_provider_by_id(
2906        &self,
2907        provider_id: &str,
2908    ) -> Option<Arc<dyn RemoteRunnerProvider>> {
2909        self.registry
2910            .remote_runner_providers
2911            .iter()
2912            .find(|provider| provider.id() == provider_id)
2913            .cloned()
2914    }
2915
2916    /// Resolve the live runner session for a thread along with its provider,
2917    /// failing clearly when the thread is not runner-bound.
2918    async fn thread_runner_session(
2919        &self,
2920        thread_id: &ThreadId,
2921    ) -> anyhow::Result<(
2922        RunnerDestination,
2923        Arc<dyn RemoteRunnerProvider>,
2924        Arc<dyn RemoteRunnerSession>,
2925    )> {
2926        let Some((destination, session)) = self.runner_session_for_thread(thread_id).await? else {
2927            anyhow::bail!("thread {thread_id} is not bound to a remote runner");
2928        };
2929        let provider = self
2930            .remote_runner_provider_by_id(&destination.provider_id)
2931            .ok_or_else(|| {
2932                anyhow::anyhow!(
2933                    "remote runner provider {:?} is not installed",
2934                    destination.provider_id
2935                )
2936            })?;
2937        Ok((destination, provider, session))
2938    }
2939
2940    /// Pause a runner-bound thread's session toward standby and persist the
2941    /// post-pause state. Errors if the provider is not pausable.
2942    pub async fn pause_thread_runner(
2943        &self,
2944        thread_id: &ThreadId,
2945    ) -> anyhow::Result<RunnerSessionState> {
2946        let (destination, provider, session) = self.thread_runner_session(thread_id).await?;
2947        anyhow::ensure!(
2948            provider.capabilities().pausable,
2949            "remote runner provider {:?} does not support pausing",
2950            destination.provider_id
2951        );
2952        let state = session.pause().await?;
2953        self.persist_runner_state(thread_id, Some(&(destination, session)))
2954            .await?;
2955        Ok(state)
2956    }
2957
2958    /// Resume (wake) a runner-bound thread's paused session and persist state.
2959    pub async fn resume_thread_runner(
2960        &self,
2961        thread_id: &ThreadId,
2962    ) -> anyhow::Result<RunnerSessionState> {
2963        let (destination, _provider, session) = self.thread_runner_session(thread_id).await?;
2964        let state = session.resume().await?;
2965        self.persist_runner_state(thread_id, Some(&(destination, session)))
2966            .await?;
2967        Ok(state)
2968    }
2969
2970    /// Detach a runner-bound thread's session: persist the durable, rejoinable
2971    /// state and leave the remote sandbox alive. Errors if not detachable.
2972    pub async fn detach_thread_runner(
2973        &self,
2974        thread_id: &ThreadId,
2975    ) -> anyhow::Result<RunnerSessionState> {
2976        let (destination, provider, session) = self.thread_runner_session(thread_id).await?;
2977        anyhow::ensure!(
2978            provider.capabilities().detachable,
2979            "remote runner provider {:?} does not support detaching",
2980            destination.provider_id
2981        );
2982        let state = session.detach().await?;
2983        // Persist the detached state explicitly so a later turn or process
2984        // rejoins the same sandbox instead of provisioning a new one.
2985        if let Some(store) = &self.thread_store
2986            && let Some(snapshot) = store.load_thread(thread_id).await?
2987            && let Some(mut metadata) = snapshot.metadata
2988        {
2989            metadata.runner_destination = Some(destination);
2990            metadata.runner_state = Some(state.clone());
2991            metadata.updated_at = OffsetDateTime::now_utc();
2992            store.update_thread_metadata(metadata).await?;
2993        }
2994        self.evict_runner_session(thread_id).await;
2995        Ok(state)
2996    }
2997
2998    /// Rejoin a previously created sandbox from a thread's persisted runner
2999    /// state without provisioning a new one. An optional `sandbox` overrides the
3000    /// persisted sandbox name (recovery by name). Persists refreshed state.
3001    pub async fn rejoin_thread_runner(
3002        &self,
3003        thread_id: &ThreadId,
3004        sandbox: Option<String>,
3005    ) -> anyhow::Result<RunnerSessionState> {
3006        let store = self
3007            .thread_store
3008            .as_ref()
3009            .ok_or_else(|| anyhow::anyhow!("thread store is required to rejoin a runner"))?;
3010        let metadata = store
3011            .load_thread_metadata(thread_id)
3012            .await?
3013            .ok_or_else(|| anyhow::anyhow!("thread {thread_id} has no metadata"))?;
3014        let destination = metadata
3015            .runner_binding
3016            .as_ref()
3017            .map(|binding| binding.destination.clone())
3018            .or_else(|| metadata.runner_destination.clone())
3019            .ok_or_else(|| anyhow::anyhow!("thread {thread_id} is not bound to a remote runner"))?;
3020        let mut state = metadata
3021            .runner_state
3022            .clone()
3023            .ok_or_else(|| anyhow::anyhow!("thread {thread_id} has no persisted runner state"))?;
3024        if let Some(sandbox) = sandbox
3025            && let Some(object) = state.metadata.as_object_mut()
3026        {
3027            object.insert("sandbox_name".to_string(), serde_json::Value::from(sandbox));
3028        }
3029        let provider = self
3030            .remote_runner_provider_by_id(&destination.provider_id)
3031            .ok_or_else(|| {
3032                anyhow::anyhow!(
3033                    "remote runner provider {:?} is not installed",
3034                    destination.provider_id
3035                )
3036            })?;
3037        let session = provider.rejoin_session(state).await?;
3038        self.persist_runner_state(thread_id, Some(&(destination.clone(), session.clone())))
3039            .await?;
3040        self.cache_runner_session(thread_id, destination, session.clone())
3041            .await;
3042        Ok(session.state())
3043    }
3044
3045    async fn record_thread_usage_metadata(
3046        &self,
3047        thread_id: &ThreadId,
3048        usage: &TokenUsage,
3049    ) -> anyhow::Result<()> {
3050        if usage.is_empty() {
3051            return Ok(());
3052        }
3053        let Some(store) = &self.thread_store else {
3054            return Ok(());
3055        };
3056        let Some(snapshot) = store.load_thread(thread_id).await? else {
3057            return Ok(());
3058        };
3059        let Some(mut metadata) = snapshot.metadata else {
3060            return Ok(());
3061        };
3062        metadata
3063            .usage
3064            .get_or_insert_with(ThreadUsageMetadata::default)
3065            .add_token_usage(usage);
3066        metadata.updated_at = OffsetDateTime::now_utc();
3067        store.update_thread_metadata(metadata).await?;
3068        Ok(())
3069    }
3070
3071    pub fn start_turn(
3072        self: &Arc<Self>,
3073        mut req: StartTurnRequest,
3074    ) -> BoxFuture<'_, anyhow::Result<TurnId>> {
3075        Box::pin(async move {
3076            let turn_admission = self.turn_admission.lock().await;
3077            anyhow::ensure!(
3078                self.accepting_turns.load(Ordering::Acquire),
3079                "runtime is quiescing and cannot accept new turns"
3080            );
3081            req.workspace = validate_thread_workspace(&req.workspace)?;
3082            let team_member = self.teams.member_for_thread(&req.thread_id).await;
3083            let cfg = self.config.read().await.clone();
3084            let provider = req
3085                .provider_override
3086                .clone()
3087                .unwrap_or_else(|| cfg.default_provider.clone());
3088            self.engine_for(&provider)?;
3089            let turn_id = uuid::Uuid::new_v4().to_string();
3090            let mut initial_mailbox_ack = None;
3091            if let Some((team_id, member)) = &team_member {
3092                let pending = self
3093                    .teams
3094                    .reserve_pending_mailbox_messages(team_id, &member.id, &turn_id)
3095                    .await?;
3096                if !pending.is_empty()
3097                    && let Some(team) = self.read_team(team_id).await
3098                {
3099                    let mailbox = format_mailbox_messages(&team, &pending);
3100                    req.message = if req.message.trim().is_empty() {
3101                        mailbox
3102                    } else {
3103                        format!("{mailbox}\n\n[Direct task input]\n{}", req.message)
3104                    };
3105                    initial_mailbox_ack = Some(MailboxDeliveryAck {
3106                        team_id: team_id.clone(),
3107                        message_ids: pending.iter().map(|message| message.id.clone()).collect(),
3108                    });
3109                }
3110            }
3111            let (abort_handle, abort_registration) = AbortHandle::new_pair();
3112            let drain = Arc::new(TurnDrainHandle {
3113                thread_id: req.thread_id.clone(),
3114                interrupt_requested: AtomicBool::new(false),
3115                interrupt_reason: Mutex::new(None),
3116                completed: AtomicBool::new(false),
3117                completed_notify: Notify::new(),
3118            });
3119            let active = ActiveTurnHandle {
3120                thread_id: req.thread_id.clone(),
3121                abort: abort_handle,
3122                steers: Arc::new(Mutex::new(Vec::new())),
3123                drain,
3124            };
3125            self.active_turns
3126                .write()
3127                .await
3128                .insert(turn_id.clone(), active);
3129            self.record_turn_lifecycle(
3130                req.thread_id.clone(),
3131                turn_id.clone(),
3132                TurnLifecycleState::Running,
3133                TurnCleanupState::NotRequested,
3134                None,
3135            )
3136            .await;
3137            self.active_turn_contexts.write().await.insert(
3138                turn_id.clone(),
3139                InheritedTurnContext {
3140                    workspace: req.workspace.clone(),
3141                    instructions: req.instructions.clone(),
3142                    developer_context: req.developer_context.clone(),
3143                },
3144            );
3145            if let Some((team_id, member)) = team_member {
3146                let updated = match self
3147                    .teams
3148                    .update_member(&team_id, &member.id, |member| {
3149                        member.current_turn_id = Some(turn_id.clone());
3150                        member.status = TeamMemberStatus::Running;
3151                        member.final_message = None;
3152                        member.terminal_error = None;
3153                    })
3154                    .await
3155                {
3156                    Ok(updated) => updated,
3157                    Err(error) => {
3158                        self.active_turns.write().await.remove(&turn_id);
3159                        self.active_turn_contexts.write().await.remove(&turn_id);
3160                        self.teams
3161                            .release_mailbox_reservations_for_turn(&turn_id)
3162                            .await;
3163                        return Err(error);
3164                    }
3165                };
3166                if let Some(member) = updated
3167                    .members
3168                    .into_iter()
3169                    .find(|candidate| candidate.id == member.id)
3170                {
3171                    self.emit(RoderEvent::TeamMemberStatusChanged(
3172                        TeamMemberStatusChanged {
3173                            team_id,
3174                            member_id: member.id,
3175                            member_thread_id: member.thread_id,
3176                            status: TeamMemberStatus::Running,
3177                            timestamp: OffsetDateTime::now_utc(),
3178                        },
3179                    ))
3180                    .await;
3181                }
3182            }
3183            let runtime = Arc::clone(self);
3184            let turn_req = req;
3185            let thread_id_for_task = turn_req.thread_id.clone();
3186            let turn_id_for_task = turn_id.clone();
3187            tokio::spawn(async move {
3188                let result = Abortable::new(
3189                    runtime.run_turn(turn_req, turn_id_for_task.clone(), initial_mailbox_ack),
3190                    abort_registration,
3191                )
3192                .await;
3193                /*
3194                 * A failed sibling in a parallel tool batch drops in-flight external tool
3195                 * futures (`try_join_all` in `route_tool_calls`), stranding their
3196                 * `pending_external_tool_calls` entries. Sweep before reporting the turn
3197                 * outcome so every `thread/toolExecutionRequested` gets a terminal
3198                 * resolution; on clean completion the map holds nothing for this turn.
3199                 */
3200                runtime
3201                    .cancel_pending_external_tool_calls_for_turn(&turn_id_for_task)
3202                    .await;
3203                let completed = matches!(&result, Ok(Ok(TurnRunOutcome::Completed)));
3204                match &result {
3205                    Ok(Err(err)) => {
3206                        let (cleanup, ownership) =
3207                            runtime.await_provider_turn_cleanup(&turn_id_for_task).await;
3208                        // run_turn emits failures after the stream starts; this covers setup/startup errors.
3209                        runtime
3210                            .emit(RoderEvent::TurnFailed(TurnFailed {
3211                                thread_id: thread_id_for_task.clone(),
3212                                turn_id: turn_id_for_task.clone(),
3213                                error: err.to_string(),
3214                                error_kind: None,
3215                                usage: None,
3216                                timestamp: OffsetDateTime::now_utc(),
3217                            }))
3218                            .await;
3219                        runtime
3220                            .record_turn_lifecycle_with_ownership(
3221                                thread_id_for_task.clone(),
3222                                turn_id_for_task.clone(),
3223                                TurnLifecycleState::Failed,
3224                                cleanup,
3225                                Some(TurnLifecycleReason::ProviderFailure),
3226                                ownership,
3227                            )
3228                            .await;
3229                        let _ = runtime
3230                            .complete_team_member_turn_with_result(
3231                                &thread_id_for_task,
3232                                &turn_id_for_task,
3233                                TeamMemberStatus::Failed,
3234                                None,
3235                                Some(err.to_string()),
3236                            )
3237                            .await;
3238                    }
3239                    Ok(Ok(TurnRunOutcome::Stopped)) => {
3240                        let (cleanup, ownership) =
3241                            runtime.await_provider_turn_cleanup(&turn_id_for_task).await;
3242                        runtime
3243                            .record_turn_lifecycle_with_ownership(
3244                                thread_id_for_task.clone(),
3245                                turn_id_for_task.clone(),
3246                                TurnLifecycleState::Failed,
3247                                cleanup,
3248                                Some(TurnLifecycleReason::ProviderFailure),
3249                                ownership,
3250                            )
3251                            .await;
3252                        let _ = runtime
3253                            .complete_team_member_turn_with_result(
3254                                &thread_id_for_task,
3255                                &turn_id_for_task,
3256                                TeamMemberStatus::Failed,
3257                                None,
3258                                Some("turn stopped before completion".to_string()),
3259                            )
3260                            .await;
3261                    }
3262                    Err(_) => {
3263                        let reason = if let Some(handle) = runtime
3264                            .turn_drains
3265                            .read()
3266                            .await
3267                            .get(&turn_id_for_task)
3268                            .cloned()
3269                        {
3270                            handle
3271                                .interrupt_reason
3272                                .lock()
3273                                .await
3274                                .unwrap_or(TurnLifecycleReason::RuntimeFailure)
3275                        } else {
3276                            TurnLifecycleReason::RuntimeFailure
3277                        };
3278                        let (cleanup, ownership) =
3279                            runtime.await_provider_turn_cleanup(&turn_id_for_task).await;
3280                        runtime
3281                            .record_turn_lifecycle_with_ownership(
3282                                thread_id_for_task.clone(),
3283                                turn_id_for_task.clone(),
3284                                TurnLifecycleState::Interrupted,
3285                                cleanup,
3286                                Some(reason),
3287                                ownership,
3288                            )
3289                            .await;
3290                        runtime
3291                            .emit(RoderEvent::TurnInterrupted(TurnInterrupted {
3292                                thread_id: thread_id_for_task.clone(),
3293                                turn_id: turn_id_for_task.clone(),
3294                                timestamp: OffsetDateTime::now_utc(),
3295                            }))
3296                            .await;
3297                        let _ = runtime
3298                            .complete_team_member_turn_with_result(
3299                                &thread_id_for_task,
3300                                &turn_id_for_task,
3301                                TeamMemberStatus::Interrupted,
3302                                None,
3303                                None,
3304                            )
3305                            .await;
3306                    }
3307                    Ok(Ok(TurnRunOutcome::Completed)) => {}
3308                }
3309                runtime
3310                    .teams
3311                    .release_mailbox_reservations_for_turn(&turn_id_for_task)
3312                    .await;
3313                runtime.active_turns.write().await.remove(&turn_id_for_task);
3314                if let Some(drain) = runtime.turn_drains.write().await.remove(&turn_id_for_task) {
3315                    drain.completed.store(true, Ordering::Release);
3316                    drain.completed_notify.notify_waiters();
3317                }
3318                runtime
3319                    .active_turn_selections
3320                    .write()
3321                    .await
3322                    .remove(&turn_id_for_task);
3323                runtime
3324                    .active_turn_contexts
3325                    .write()
3326                    .await
3327                    .remove(&turn_id_for_task);
3328                if !completed {
3329                    // Completion paths above consume registered provider cleanup
3330                    // handles. A setup failure before an engine can stream has no
3331                    // such handle; this is a harmless final defensive sweep.
3332                    let _ = runtime.provider_turn_cleanups.lock().map(|mut cleanups| {
3333                        cleanups.remove(&turn_id_for_task);
3334                    });
3335                }
3336                runtime.active_turns_changed.notify_waiters();
3337                if completed {
3338                    let _ = runtime
3339                        .continue_active_goal_after_turn(thread_id_for_task)
3340                        .await;
3341                }
3342            });
3343            drop(turn_admission);
3344            Ok(turn_id)
3345        })
3346    }
3347
3348    pub(crate) async fn has_active_turn_for_thread(&self, thread_id: &ThreadId) -> bool {
3349        self.active_turns
3350            .read()
3351            .await
3352            .values()
3353            .any(|handle| &handle.thread_id == thread_id)
3354    }
3355
3356    async fn ensure_codex_v2_team_capacity(
3357        &self,
3358        team: &TeamState,
3359        resuming_member_id: &str,
3360        candidate_model: Option<&str>,
3361    ) -> anyhow::Result<()> {
3362        if !is_codex_v2_team(team)
3363            && !candidate_model
3364                .is_some_and(|model| model_supports_reasoning_effort(model, REASONING_ULTRA))
3365        {
3366            return Ok(());
3367        }
3368        let active_thread_ids = self
3369            .active_turns
3370            .read()
3371            .await
3372            .values()
3373            .map(|handle| handle.thread_id.clone())
3374            .collect::<std::collections::HashSet<_>>();
3375        let resident_threads = 1 + team
3376            .members
3377            .iter()
3378            .filter(|member| {
3379                member.role != roder_api::teams::TeamMemberRole::Lead
3380                    && member.id != resuming_member_id
3381                    && active_thread_ids.contains(&member.thread_id)
3382            })
3383            .count();
3384        anyhow::ensure!(
3385            resident_threads < codex_v2::CODEX_V2_MAX_RESIDENT_TEAM_THREADS,
3386            "agent thread limit reached for this Codex V2 team: maximum {} resident threads (the lead plus 3 running subagents)",
3387            codex_v2::CODEX_V2_MAX_RESIDENT_TEAM_THREADS
3388        );
3389        Ok(())
3390    }
3391
3392    /// Number of currently running turns across all threads. Hosted runtime
3393    /// pools use this to avoid evicting tenants with active work.
3394    pub async fn active_turn_count(&self) -> usize {
3395        self.active_turns.read().await.len()
3396    }
3397
3398    pub async fn active_turn_for_thread(&self, thread_id: &ThreadId) -> Option<TurnId> {
3399        self.active_turns
3400            .read()
3401            .await
3402            .iter()
3403            .find_map(|(turn_id, handle)| (&handle.thread_id == thread_id).then(|| turn_id.clone()))
3404    }
3405
3406    pub async fn thread_activity(&self, thread_id: &ThreadId) -> ThreadActivity {
3407        let active_turn_id = self.active_turn_for_thread(thread_id).await;
3408        let draining_turn_id = if active_turn_id.is_none() {
3409            self.turn_drains
3410                .read()
3411                .await
3412                .iter()
3413                .find_map(|(turn_id, drain)| {
3414                    (&drain.thread_id == thread_id).then(|| turn_id.clone())
3415                })
3416        } else {
3417            None
3418        };
3419        let Some(active_turn_id) = active_turn_id.or(draining_turn_id) else {
3420            return ThreadActivity::default();
3421        };
3422
3423        let mut active_flags = Vec::new();
3424        {
3425            let pending_approvals = self.pending_tool_approvals.lock().await;
3426            if pending_approvals
3427                .values()
3428                .any(|pending| &pending.thread_id == thread_id && pending.turn_id == active_turn_id)
3429            {
3430                active_flags.push("approvalRequired".to_string());
3431            }
3432        }
3433        {
3434            let pending_inputs = self.pending_user_inputs.lock().await;
3435            if pending_inputs
3436                .values()
3437                .any(|pending| &pending.thread_id == thread_id && pending.turn_id == active_turn_id)
3438            {
3439                active_flags.push("userInputRequired".to_string());
3440            }
3441        }
3442        {
3443            let pending_external = self.pending_external_tool_calls.lock().await;
3444            if pending_external
3445                .values()
3446                .any(|pending| &pending.thread_id == thread_id && pending.turn_id == active_turn_id)
3447            {
3448                active_flags.push("externalToolPending".to_string());
3449            }
3450        }
3451        let interrupting = self
3452            .active_turns
3453            .read()
3454            .await
3455            .get(&active_turn_id)
3456            .is_some_and(|handle| handle.drain.interrupt_requested.load(Ordering::Acquire))
3457            || self
3458                .turn_drains
3459                .read()
3460                .await
3461                .get(&active_turn_id)
3462                .is_some_and(|drain| drain.interrupt_requested.load(Ordering::Acquire));
3463        if interrupting {
3464            active_flags.push("interrupting".to_string());
3465        }
3466        if self.pending_plan_exit().await.is_some_and(|pending| {
3467            &pending.thread_id == thread_id && pending.turn_id == active_turn_id
3468        }) {
3469            active_flags.push("planExitRequired".to_string());
3470        }
3471
3472        ThreadActivity {
3473            active_turn_id: Some(active_turn_id),
3474            active_flags,
3475        }
3476    }
3477
3478    pub async fn interrupt_turn(&self, thread_id: ThreadId, turn_id: TurnId) -> anyhow::Result<()> {
3479        self.interrupt_turn_with_reason(thread_id, turn_id, TurnLifecycleReason::UserInterrupt)
3480            .await
3481    }
3482
3483    async fn interrupt_turn_with_reason(
3484        &self,
3485        thread_id: ThreadId,
3486        turn_id: TurnId,
3487        reason: TurnLifecycleReason,
3488    ) -> anyhow::Result<()> {
3489        let handle = self.active_turns.write().await.remove(&turn_id);
3490        if let Some(handle) = handle {
3491            if handle
3492                .drain
3493                .interrupt_requested
3494                .swap(true, Ordering::AcqRel)
3495            {
3496                return Ok(());
3497            }
3498            *handle.drain.interrupt_reason.lock().await = Some(reason);
3499            self.turn_drains
3500                .write()
3501                .await
3502                .insert(turn_id.clone(), handle.drain.clone());
3503            let ownership = self.provider_cleanup_ownership(&turn_id);
3504            self.record_turn_lifecycle_with_ownership(
3505                thread_id.clone(),
3506                turn_id.clone(),
3507                TurnLifecycleState::InterruptRequested,
3508                TurnCleanupState::Requested,
3509                Some(reason),
3510                ownership,
3511            )
3512            .await;
3513            handle.abort.abort();
3514            self.active_turns_changed.notify_waiters();
3515        }
3516        self.active_turn_selections.write().await.remove(&turn_id);
3517        self.cancel_pending_external_tool_calls_for_turn(&turn_id)
3518            .await;
3519        Ok(())
3520    }
3521
3522    pub async fn steer_turn(
3523        &self,
3524        thread_id: ThreadId,
3525        turn_id: TurnId,
3526        message: String,
3527        images: Vec<InputImage>,
3528    ) -> anyhow::Result<()> {
3529        self.enqueue_turn_steer(thread_id, turn_id, message, images, None)
3530            .await
3531    }
3532
3533    pub(crate) async fn has_pending_turn_steers(&self, turn_id: &TurnId) -> bool {
3534        let active = self.active_turns.read().await.get(turn_id).cloned();
3535        let Some(active) = active else {
3536            return false;
3537        };
3538        let has_pending = !active.steers.lock().await.is_empty();
3539        has_pending
3540    }
3541
3542    async fn steer_turn_with_mailbox_ack(
3543        &self,
3544        thread_id: ThreadId,
3545        turn_id: TurnId,
3546        message: String,
3547        message_ids: Vec<String>,
3548        team_id: TeamId,
3549    ) -> anyhow::Result<()> {
3550        self.enqueue_turn_steer(
3551            thread_id,
3552            turn_id,
3553            message,
3554            Vec::new(),
3555            Some(MailboxDeliveryAck {
3556                team_id,
3557                message_ids,
3558            }),
3559        )
3560        .await
3561    }
3562
3563    async fn deliver_pending_team_mailbox(
3564        &self,
3565        team_id: &str,
3566        member_id: &str,
3567        member_thread_id: ThreadId,
3568        turn_id: TurnId,
3569    ) -> anyhow::Result<()> {
3570        let pending = self
3571            .teams
3572            .reserve_pending_mailbox_messages(team_id, member_id, &turn_id)
3573            .await?;
3574        if pending.is_empty() {
3575            return Ok(());
3576        }
3577        let message_ids = pending
3578            .iter()
3579            .map(|message| message.id.clone())
3580            .collect::<Vec<_>>();
3581        let team = self
3582            .read_team(team_id)
3583            .await
3584            .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
3585        if let Err(error) = self
3586            .steer_turn_with_mailbox_ack(
3587                member_thread_id,
3588                turn_id.clone(),
3589                format_mailbox_messages(&team, &pending),
3590                message_ids.clone(),
3591                team_id.to_string(),
3592            )
3593            .await
3594        {
3595            self.teams
3596                .release_mailbox_reservations(&turn_id, &message_ids)
3597                .await;
3598            return Err(error);
3599        }
3600        Ok(())
3601    }
3602
3603    async fn enqueue_turn_steer(
3604        &self,
3605        thread_id: ThreadId,
3606        turn_id: TurnId,
3607        message: String,
3608        images: Vec<InputImage>,
3609        mailbox_ack: Option<MailboxDeliveryAck>,
3610    ) -> anyhow::Result<()> {
3611        let message = message.trim().to_string();
3612        if message.is_empty() && images.is_empty() {
3613            return Ok(());
3614        }
3615
3616        let Some(active) = self.active_turns.read().await.get(&turn_id).cloned() else {
3617            anyhow::bail!("no active turn to steer");
3618        };
3619        active.steers.lock().await.push(QueuedTurnSteer {
3620            message: UserMessage::with_images(message.clone(), images),
3621            mailbox_ack,
3622        });
3623        self.emit(RoderEvent::TurnSteered(TurnSteered {
3624            thread_id,
3625            turn_id,
3626            message,
3627            timestamp: OffsetDateTime::now_utc(),
3628        }))
3629        .await;
3630        Ok(())
3631    }
3632
3633    pub async fn tool_specs(&self) -> Vec<roder_api::tools::ToolSpec> {
3634        let cfg = self.config.read().await;
3635        let model_profile =
3636            model_profile_for_provider_model(&cfg, &cfg.default_provider, &cfg.default_model);
3637        self.filtered_tool_specs(&cfg, &cfg.default_model, model_profile.as_ref(), &[], &[])
3638    }
3639
3640    pub fn subagent_definitions(&self) -> Vec<SubagentDefinition> {
3641        self.registry
3642            .subagent_dispatchers
3643            .iter()
3644            .flat_map(|dispatcher| dispatcher.definitions())
3645            .collect()
3646    }
3647
3648    async fn run_turn(
3649        self: &Arc<Self>,
3650        req: StartTurnRequest,
3651        turn_id: TurnId,
3652        initial_mailbox_ack: Option<MailboxDeliveryAck>,
3653    ) -> anyhow::Result<TurnRunOutcome> {
3654        let turn_started_at = OffsetDateTime::now_utc();
3655        self.emit(RoderEvent::TurnStarted(TurnStarted {
3656            thread_id: req.thread_id.clone(),
3657            turn_id: turn_id.clone(),
3658            runtime_profile: self.config.read().await.runtime_profile,
3659            timestamp: turn_started_at,
3660        }))
3661        .await;
3662        self.persist_turn_item(
3663            &req.thread_id,
3664            &turn_id,
3665            &TranscriptItem::UserMessage(UserMessage::with_images(
3666                req.message.clone(),
3667                req.images.clone(),
3668            )),
3669        )
3670        .await?;
3671        if let Some(ack) = initial_mailbox_ack {
3672            self.teams
3673                .mark_mailbox_messages_delivered(&ack.team_id, &turn_id, &ack.message_ids)
3674                .await?;
3675        }
3676
3677        let mut cfg = self.config.read().await.clone();
3678        let runtime_profile = cfg.runtime_profile;
3679        let turn_deadline = turn_deadline_for_config(&cfg);
3680        let deadline_finalization_reserve =
3681            crate::deadline_policy::finalization_reserve_seconds(cfg.turn_deadline_seconds);
3682        let selection_mode = self.selection_mode_for_thread(&req.thread_id).await?;
3683        let concrete_selection = selection_mode
3684            .as_ref()
3685            .map(ModelSelectionMode::concrete_selection);
3686        let default_provider = req
3687            .provider_override
3688            .clone()
3689            .or_else(|| {
3690                concrete_selection
3691                    .as_ref()
3692                    .map(|selection| selection.provider.clone())
3693            })
3694            .unwrap_or(cfg.default_provider.clone());
3695        let default_model = req
3696            .model_override
3697            .clone()
3698            .or_else(|| {
3699                concrete_selection
3700                    .as_ref()
3701                    .map(|selection| selection.model.clone())
3702            })
3703            .unwrap_or(cfg.default_model.clone());
3704        if let Some(reasoning) = req.reasoning_override.as_deref().or_else(|| {
3705            selection_mode
3706                .as_ref()
3707                .and_then(ModelSelectionMode::reasoning)
3708        }) {
3709            validate_reasoning_effort(&default_model, reasoning)?;
3710            cfg.reasoning = Some(reasoning.to_string());
3711        }
3712        let turn_has_concrete_model_override =
3713            req.provider_override.is_some() || req.model_override.is_some();
3714        let (turn_inference_router, turn_inference_router_profile) = match &selection_mode {
3715            Some(ModelSelectionMode::Auto {
3716                router_id, profile, ..
3717            }) if !turn_has_concrete_model_override => (
3718                RuntimeInferenceRouterConfig {
3719                    enabled: true,
3720                    router_id: Some(router_id.clone()),
3721                },
3722                profile.clone(),
3723            ),
3724            _ => (RuntimeInferenceRouterConfig::disabled(), None),
3725        };
3726        let mut provider = default_provider.clone();
3727        let mut model = default_model.clone();
3728        let mut model_profile = model_profile_for_provider_model(&cfg, &provider, &model);
3729        let workspace = req.workspace.clone();
3730        let mut transcript = self.transcript_for_turn(&req, &turn_id, &model).await?;
3731        let mut compacted_this_turn = transcript
3732            .iter()
3733            .any(crate::compaction::is_compaction_boundary);
3734        let effective_policy_mode = self.effective_policy_mode_for_thread(&req.thread_id).await;
3735        let agent_swarm_mode_active = self
3736            .effective_agent_swarm_mode_for_thread(&req.thread_id)
3737            .await;
3738        let thread_overrides = self.thread_turn_overrides(&req.thread_id).await?;
3739        let mut final_assistant_text = String::new();
3740        let mut final_phase_messages = Vec::<AssistantMessage>::new();
3741        let mut final_reasoning_text = String::new();
3742        let mut final_provider_metadata = None;
3743        let mut exhausted_tool_rounds = true;
3744        let mut verification_gate =
3745            VerificationGateState::new(req.message.clone(), runtime_profile);
3746        let mut speed_policy = SpeedPolicyState::default();
3747        let mut reliability = TurnReliabilityState::default();
3748        let mut turn_usage = TokenUsage::default();
3749        // Overwritten on every inference step's Completed event so only the
3750        // terminal step's stop reason survives (mid-turn tool_use steps must
3751        // not leak onto turn/completed).
3752        let mut turn_finish_reason: Option<String> = None;
3753        let mut deadline_finalization_requested = false;
3754        let mut deadline_scoreable_completion_requested = false;
3755        let mut task_ledger_completion_reminders = 0_u8;
3756        let mut task_ledger_scoreable_checkpoints = 0_u8;
3757        let mut empty_tool_call_nudges_used = 0_u32;
3758        let mut provider_stream_retry_attempts = 0_u32;
3759        let mut routing_candidates = None;
3760        let routing_transcript_start = transcript.len().saturating_sub(1);
3761        let mut routing_escalations = 0_u32;
3762        let mut model_switch_summary_selection = None::<ModelSelection>;
3763
3764        'tool_rounds: for round_index in 0..MAX_TOOL_ROUNDS_PER_TURN {
3765            if let Some(deadline) = turn_deadline
3766                && deadline_expired(deadline)
3767            {
3768                self.fail_turn_due_to_deadline(&req.thread_id, &turn_id, deadline, &transcript)
3769                    .await?;
3770                return Ok(TurnRunOutcome::Stopped);
3771            }
3772            let steers = self.drain_turn_steers(&turn_id).await;
3773            self.append_steers(&req, &turn_id, &mut transcript, steers)
3774                .await?;
3775            if runtime_profile == RuntimeProfile::Eval
3776                && let Some(remaining) = crate::deadline_policy::should_start_finalization(
3777                    turn_deadline,
3778                    deadline_finalization_reserve,
3779                    deadline_finalization_requested || deadline_scoreable_completion_requested,
3780                )
3781            {
3782                if req.task_ledger_required
3783                    && task_ledger_completion_reminders < TASK_LEDGER_COMPLETION_REMINDER_LIMIT
3784                    && let Some(prompt) = task_ledger_completion_prompt(&transcript)
3785                {
3786                    task_ledger_completion_reminders += 1;
3787                    deadline_scoreable_completion_requested = true;
3788                    let item = TranscriptItem::UserMessage(UserMessage::text(
3789                        task_ledger_deadline_completion_prompt(
3790                            remaining,
3791                            deadline_finalization_reserve,
3792                            &prompt,
3793                        ),
3794                    ));
3795                    self.persist_turn_item(&req.thread_id, &turn_id, &item)
3796                        .await?;
3797                    transcript.push(item);
3798                    continue 'tool_rounds;
3799                } else {
3800                    self.start_deadline_finalization(
3801                        &req.thread_id,
3802                        &turn_id,
3803                        &mut transcript,
3804                        remaining,
3805                    )
3806                    .await?;
3807                    deadline_finalization_requested = true;
3808                }
3809            }
3810            if runtime_profile == RuntimeProfile::Eval
3811                && req.task_ledger_required
3812                && !deadline_finalization_requested
3813                && task_ledger_scoreable_checkpoints < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
3814                && let Some(remaining) = deadline_remaining_seconds(turn_deadline)
3815                && remaining <= TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS
3816                && remaining > deadline_finalization_reserve
3817                && let Some(prompt) = task_ledger_completion_prompt(&transcript)
3818            {
3819                task_ledger_scoreable_checkpoints += 1;
3820                let item = TranscriptItem::UserMessage(UserMessage::text(
3821                    task_ledger_scoreable_checkpoint_prompt(remaining, &prompt),
3822                ));
3823                self.persist_turn_item(&req.thread_id, &turn_id, &item)
3824                    .await?;
3825                transcript.push(item);
3826                continue 'tool_rounds;
3827            }
3828            if turn_inference_router.is_active() && routing_candidates.is_none() {
3829                routing_candidates =
3830                    Some(collect_inference_routing_candidates(&self.registry).await);
3831            }
3832            let routing_tools_model = model.clone();
3833            let routing_tools = self.filtered_tool_specs(
3834                &cfg,
3835                &model,
3836                model_profile.as_ref(),
3837                &thread_overrides.tool_allowlist,
3838                &thread_overrides.external_tools,
3839            );
3840            let prior_failures =
3841                transcript_failure_count_since(&transcript, routing_transcript_start)
3842                    .max(reliability.tool_failure_count())
3843                    .saturating_add(provider_stream_retry_attempts);
3844            let routing_selection = route_inference_selection(
3845                &self.registry,
3846                &turn_inference_router,
3847                InferenceRoutingRequest {
3848                    thread_id: &req.thread_id,
3849                    turn_id: &turn_id,
3850                    round_index: round_index as u32,
3851                    runtime_profile,
3852                    phase: speed_policy.phase(),
3853                    profile: turn_inference_router_profile.as_deref(),
3854                    default_selection: ModelSelection {
3855                        provider: default_provider.clone(),
3856                        model: default_model.clone(),
3857                    },
3858                    transcript: &transcript,
3859                    tools: &routing_tools,
3860                    candidates: routing_candidates.as_deref(),
3861                    prior_failures,
3862                    prior_escalations: routing_escalations,
3863                },
3864            )
3865            .await;
3866            if let Some(decision) = routing_selection.decision.clone() {
3867                if matches!(decision.outcome, InferenceRoutingOutcome::Escalated) {
3868                    routing_escalations = routing_escalations.saturating_add(1);
3869                }
3870                self.emit(RoderEvent::InferenceRoutingDecision(
3871                    InferenceRoutingDecisionEvent {
3872                        thread_id: req.thread_id.clone(),
3873                        turn_id: turn_id.clone(),
3874                        round_index: round_index as u32,
3875                        default_selection: ModelSelection {
3876                            provider: default_provider.clone(),
3877                            model: default_model.clone(),
3878                        },
3879                        selected_selection: routing_selection.selection.clone(),
3880                        decision,
3881                        timestamp: OffsetDateTime::now_utc(),
3882                    },
3883                ))
3884                .await;
3885            }
3886            provider = routing_selection.selection.provider.clone();
3887            model = routing_selection.selection.model.clone();
3888            let engine = self.engine_for(&provider)?;
3889            let capabilities = engine.capabilities();
3890            model_profile = model_profile_for_provider_model(&cfg, &provider, &model);
3891            let tools = if capabilities.tool_calls {
3892                if model == routing_tools_model {
3893                    routing_tools.clone()
3894                } else {
3895                    self.filtered_tool_specs(
3896                        &cfg,
3897                        &model,
3898                        model_profile.as_ref(),
3899                        &thread_overrides.tool_allowlist,
3900                        &thread_overrides.external_tools,
3901                    )
3902                }
3903            } else {
3904                Vec::new()
3905            };
3906            let parallel_tool_calls = parallel_tool_calls_for_model(&cfg, &model);
3907            let tool_choice = if tools.is_empty() {
3908                ToolChoice::None
3909            } else {
3910                ToolChoice::Auto
3911            };
3912            let summary_selection = ModelSelection {
3913                provider: provider.clone(),
3914                model: model.clone(),
3915            };
3916            if model_switch_summary_selection.as_ref() != Some(&summary_selection) {
3917                if let Some(summary) = model_switch_summary(
3918                    &transcript,
3919                    model_profile.as_ref(),
3920                    &provider,
3921                    &model,
3922                    &tools,
3923                ) {
3924                    let item = TranscriptItem::UserMessage(UserMessage::text(summary));
3925                    self.persist_turn_item(&req.thread_id, &turn_id, &item)
3926                        .await?;
3927                    transcript.push(item);
3928                }
3929                model_switch_summary_selection = Some(summary_selection);
3930            }
3931
3932            if !capabilities.image_input && transcript_has_images(&transcript) {
3933                self.fail_turn_with_error(
3934                    &req.thread_id,
3935                    &turn_id,
3936                    format!("provider {provider} does not support image input"),
3937                )
3938                .await?;
3939                return Ok(TurnRunOutcome::Stopped);
3940            }
3941            transcript = self
3942                .compact_transcript_if_needed(
3943                    &req.thread_id,
3944                    &turn_id,
3945                    &provider,
3946                    &model,
3947                    transcript,
3948                    self.compaction_options_for_turn(&req.thread_id, !compacted_this_turn),
3949                )
3950                .await?;
3951            compacted_this_turn = compacted_this_turn
3952                || transcript
3953                    .iter()
3954                    .any(crate::compaction::is_compaction_boundary);
3955
3956            let speed_policy_decision =
3957                speed_policy.decision(runtime_profile, &model, &cfg.speed_policy);
3958            let request_reasoning = reasoning_from_decision(
3959                speed_policy_decision.as_ref(),
3960                routing_selection
3961                    .reasoning
3962                    .clone()
3963                    .unwrap_or_else(|| reasoning_for_model(&cfg, &model)),
3964            );
3965            self.active_turn_selections.write().await.insert(
3966                turn_id.clone(),
3967                ModelSelectionMode::manual(
3968                    provider.clone(),
3969                    model.clone(),
3970                    request_reasoning.level.clone(),
3971                ),
3972            );
3973            if let Some(limit) = reliability.record_model_call(
3974                &cfg.reliability,
3975                runtime_profile == RuntimeProfile::Interactive,
3976            ) {
3977                self.fail_turn_due_to_reliability_limit(
3978                    &req.thread_id,
3979                    &turn_id,
3980                    &provider,
3981                    &model,
3982                    limit,
3983                    &transcript,
3984                )
3985                .await?;
3986                return Ok(TurnRunOutcome::Stopped);
3987            }
3988            self.emit(RoderEvent::InferenceStarted(InferenceStarted {
3989                thread_id: req.thread_id.clone(),
3990                turn_id: turn_id.clone(),
3991                engine_id: engine.id(),
3992                model: ModelSelection {
3993                    provider: provider.clone(),
3994                    model: model.clone(),
3995                },
3996                reasoning: request_reasoning.clone(),
3997                speed_policy: speed_policy_decision.clone(),
3998                deadline_remaining_seconds: deadline_remaining_seconds(turn_deadline),
3999                timestamp: OffsetDateTime::now_utc(),
4000            }))
4001            .await;
4002
4003            let mut instructions = req.instructions.clone();
4004            if let Some(extra) = &thread_overrides.developer_instructions {
4005                instructions = apply_thread_developer_instructions(instructions, extra);
4006            }
4007            if let Some(context) = req.developer_context.as_deref() {
4008                instructions = apply_turn_developer_context(instructions, context);
4009            }
4010            let mut instructions = apply_runtime_profile(instructions, runtime_profile);
4011            if let Some(profile) = &model_profile {
4012                instructions = apply_model_instruction_overlay(instructions, profile);
4013            }
4014            if req.task_ledger_required
4015                && runtime_profile == RuntimeProfile::Eval
4016                && !transcript_has_task_ledger(&transcript)
4017            {
4018                instructions = apply_task_ledger_required(instructions);
4019            }
4020            if effective_policy_mode == PolicyMode::Plan {
4021                instructions = apply_plan_mode(instructions);
4022            }
4023            if agent_swarm_mode_active {
4024                instructions = apply_agent_swarm_mode(instructions);
4025            }
4026            if model_supports_reasoning_effort(&model, REASONING_ULTRA) {
4027                instructions = apply_codex_multi_agent_mode(
4028                    instructions,
4029                    request_reasoning.level.as_deref() == Some(REASONING_ULTRA),
4030                );
4031            }
4032            instructions = self
4033                .goals
4034                .apply_goal_instructions(&req.thread_id, instructions)
4035                .await?;
4036            instructions = apply_parallel_web_tools(
4037                instructions,
4038                tools.iter().map(|spec| spec.name.as_str()),
4039            );
4040            let mut request_metadata = serde_json::json!({});
4041            if let Some(decision) = &speed_policy_decision {
4042                request_metadata["speedPolicy"] = serde_json::json!(decision);
4043            }
4044            if let Some(decision) = routing_selection.decision.as_ref() {
4045                request_metadata["inferenceRouting"] = serde_json::json!(decision);
4046            }
4047            if let Some(remaining) = deadline_remaining_seconds(turn_deadline) {
4048                request_metadata["deadlineRemainingSeconds"] = serde_json::json!(remaining);
4049            }
4050            if let Some(profile) = &model_profile {
4051                request_metadata["modelProfile"] = serde_json::json!({
4052                    "model": profile.model,
4053                    "providerFamily": profile.provider_family,
4054                    "editTool": profile.edit_tool,
4055                    "schemaPolicy": profile.schema_policy,
4056                    "instructionOverlay": profile.instruction_overlay,
4057                    "parallelToolCalls": profile.parallel_tool_calls,
4058                    "autoCompactTokenLimit": profile.auto_compact_token_limit,
4059                });
4060            }
4061            let task_ledger_required_this_round = req.task_ledger_required
4062                && runtime_profile == RuntimeProfile::Eval
4063                && !deadline_finalization_requested
4064                && !transcript_has_task_ledger(&transcript);
4065            let task_ledger_tools = (capabilities.tool_calls && task_ledger_required_this_round)
4066                .then(|| self.task_ledger_tool_specs(model_profile.as_ref()))
4067                .filter(|tools| !tools.is_empty());
4068            let request_tools = if deadline_finalization_requested {
4069                Vec::new()
4070            } else if let Some(ledger_tools) = &task_ledger_tools {
4071                ledger_tools.clone()
4072            } else {
4073                tools.clone()
4074            };
4075            let request_tool_choice = if deadline_finalization_requested {
4076                ToolChoice::None
4077            } else if task_ledger_tools.is_some() {
4078                ToolChoice::Specific(TASK_LEDGER_TOOL_NAME.to_string())
4079            } else {
4080                tool_choice.clone()
4081            };
4082            if deadline_finalization_requested {
4083                request_metadata["deadlineFinalization"] = serde_json::json!({
4084                    "reserveSeconds": deadline_finalization_reserve,
4085                    "remainingSeconds": deadline_remaining_seconds(turn_deadline),
4086                });
4087            } else if deadline_scoreable_completion_requested {
4088                request_metadata["deadlineScoreableCompletion"] = serde_json::json!({
4089                    "reserveSeconds": deadline_finalization_reserve,
4090                    "remainingSeconds": deadline_remaining_seconds(turn_deadline),
4091                });
4092            }
4093            let request = AgentInferenceRequest {
4094                model: ModelSelection {
4095                    provider: provider.clone(),
4096                    model: model.clone(),
4097                },
4098                instructions,
4099                transcript: transcript.clone(),
4100                tools: request_tools,
4101                tool_choice: request_tool_choice,
4102                reasoning: request_reasoning,
4103                output: OutputConfig::default(),
4104                runtime: RuntimeHints {
4105                    auto_compact_token_limit: server_side_compaction_threshold(&cfg, &model),
4106                    profile: runtime_profile,
4107                    parallel_tool_calls: Some(parallel_tool_calls),
4108                    hosted_web_search: cfg.hosted_web_search.clone(),
4109                    tool_search: tool_search_for_provider_model(&cfg, &provider, &model),
4110                    speed_policy: speed_policy_decision,
4111                    reliability: Some(cfg.reliability.clone().into()),
4112                    deadline_remaining_seconds: deadline_remaining_seconds(turn_deadline),
4113                    ..RuntimeHints::default()
4114                },
4115                metadata: request_metadata,
4116            };
4117
4118            let ctx = InferenceTurnContext {
4119                thread_id: &req.thread_id,
4120                turn_id: &turn_id,
4121                tool_executor: Some(std::sync::Arc::new(
4122                    crate::tool_execution::RuntimeTurnToolExecutor {
4123                        runtime: Arc::clone(self),
4124                        thread_id: req.thread_id.clone(),
4125                        turn_id: turn_id.clone(),
4126                        workspace: Some(workspace.clone()),
4127                        deadline: turn_deadline,
4128                    },
4129                )),
4130            };
4131            let stream_future = engine.stream_turn(ctx, request);
4132            let mut stream = if let Some((deadline, timeout_action)) = inference_timeout_deadline(
4133                turn_deadline,
4134                runtime_profile,
4135                req.task_ledger_required,
4136                deadline_finalization_reserve,
4137                deadline_finalization_requested || deadline_scoreable_completion_requested,
4138                task_ledger_scoreable_checkpoints,
4139                &transcript,
4140            ) {
4141                match tokio::time::timeout_at(deadline_instant(deadline), stream_future).await {
4142                    Ok(stream) => stream?,
4143                    Err(_) => {
4144                        if runtime_profile == RuntimeProfile::Eval
4145                            && !deadline_finalization_requested
4146                        {
4147                            let remaining = deadline_remaining_seconds(turn_deadline).unwrap_or(0);
4148                            if timeout_action == InferenceTimeoutAction::ScoreableCheckpoint
4149                                && task_ledger_scoreable_checkpoints
4150                                    < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
4151                                && let Some(prompt) = task_ledger_completion_prompt(&transcript)
4152                            {
4153                                task_ledger_scoreable_checkpoints += 1;
4154                                let item = TranscriptItem::UserMessage(UserMessage::text(
4155                                    task_ledger_scoreable_checkpoint_prompt(remaining, &prompt),
4156                                ));
4157                                self.persist_turn_item(&req.thread_id, &turn_id, &item)
4158                                    .await?;
4159                                transcript.push(item);
4160                                continue 'tool_rounds;
4161                            }
4162                            self.start_deadline_finalization(
4163                                &req.thread_id,
4164                                &turn_id,
4165                                &mut transcript,
4166                                remaining,
4167                            )
4168                            .await?;
4169                            deadline_finalization_requested = true;
4170                            continue 'tool_rounds;
4171                        }
4172                        self.fail_turn_due_to_deadline(
4173                            &req.thread_id,
4174                            &turn_id,
4175                            deadline,
4176                            &transcript,
4177                        )
4178                        .await?;
4179                        return Ok(TurnRunOutcome::Stopped);
4180                    }
4181                }
4182            } else {
4183                stream_future.await?
4184            };
4185            let mut assistant_text = String::new();
4186            let mut phase_messages = Vec::<AssistantMessage>::new();
4187            let mut reasoning_text = String::new();
4188            let mut tool_calls = Vec::new();
4189            let mut provider_metadata = None;
4190            // Captured from mid-stream Compaction(completed) events. Codex/OpenAI
4191            // sometimes abort the SSE stream after emitting the compaction item
4192            // ("error decoding response body") before response.completed, so we
4193            // must not rely solely on ProviderMetadata for the boundary.
4194            let mut stream_compaction_item: Option<serde_json::Value> = None;
4195
4196            loop {
4197                let next = if let Some((deadline, timeout_action)) = inference_timeout_deadline(
4198                    turn_deadline,
4199                    runtime_profile,
4200                    req.task_ledger_required,
4201                    deadline_finalization_reserve,
4202                    deadline_finalization_requested || deadline_scoreable_completion_requested,
4203                    task_ledger_scoreable_checkpoints,
4204                    &transcript,
4205                ) {
4206                    match tokio::time::timeout_at(deadline_instant(deadline), stream.next()).await {
4207                        Ok(next) => next,
4208                        Err(_) => {
4209                            if runtime_profile == RuntimeProfile::Eval
4210                                && !deadline_finalization_requested
4211                            {
4212                                let remaining =
4213                                    deadline_remaining_seconds(turn_deadline).unwrap_or(0);
4214                                if timeout_action == InferenceTimeoutAction::ScoreableCheckpoint
4215                                    && task_ledger_scoreable_checkpoints
4216                                        < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
4217                                    && let Some(prompt) = task_ledger_completion_prompt(&transcript)
4218                                {
4219                                    task_ledger_scoreable_checkpoints += 1;
4220                                    let item = TranscriptItem::UserMessage(UserMessage::text(
4221                                        task_ledger_scoreable_checkpoint_prompt(remaining, &prompt),
4222                                    ));
4223                                    self.persist_turn_item(&req.thread_id, &turn_id, &item)
4224                                        .await?;
4225                                    transcript.push(item);
4226                                    continue 'tool_rounds;
4227                                }
4228                                self.start_deadline_finalization(
4229                                    &req.thread_id,
4230                                    &turn_id,
4231                                    &mut transcript,
4232                                    remaining,
4233                                )
4234                                .await?;
4235                                deadline_finalization_requested = true;
4236                                continue 'tool_rounds;
4237                            }
4238                            self.fail_turn_due_to_deadline(
4239                                &req.thread_id,
4240                                &turn_id,
4241                                deadline,
4242                                &transcript,
4243                            )
4244                            .await?;
4245                            return Ok(TurnRunOutcome::Stopped);
4246                        }
4247                    }
4248                } else {
4249                    stream.next().await
4250                };
4251                let Some(res) = next else {
4252                    break;
4253                };
4254                let event = match res {
4255                    Ok(event) => event,
4256                    Err(err) => {
4257                        let error = err.to_string();
4258                        if runtime_profile == RuntimeProfile::Eval
4259                            && !deadline_finalization_requested
4260                            && let Some(cause) = provider_stream_retry_cause(&error)
4261                        {
4262                            let retry_attempt = provider_stream_retry_attempts.saturating_add(1);
4263                            let policy: ReliabilityRequestPolicy = cfg.reliability.clone().into();
4264                            if retry_attempt < policy.provider_retry_max_attempts {
4265                                provider_stream_retry_attempts = retry_attempt;
4266                                let delay_ms = provider_retry_delay_ms(&policy, retry_attempt);
4267                                self.emit(RoderEvent::ReliabilityRetryRecorded(
4268                                    ReliabilityRetryRecorded {
4269                                        context: ReliabilityContext {
4270                                            thread_id: req.thread_id.clone(),
4271                                            turn_id: turn_id.clone(),
4272                                            provider: Some(provider.clone()),
4273                                            model: Some(model.clone()),
4274                                            ..ReliabilityContext::default()
4275                                        },
4276                                        error_class: ReliabilityErrorClass::ProviderError,
4277                                        decision: ReliabilityRetryDecision::Retry,
4278                                        attempt: retry_attempt,
4279                                        max_attempts: policy.provider_retry_max_attempts,
4280                                        delay_ms: Some(delay_ms),
4281                                        details: ReliabilityDetails::redacted(format!(
4282                                            "{cause}: {error}"
4283                                        )),
4284                                        timestamp: OffsetDateTime::now_utc(),
4285                                    },
4286                                ))
4287                                .await;
4288                                if delay_ms > 0 {
4289                                    tokio::time::sleep(std::time::Duration::from_millis(delay_ms))
4290                                        .await;
4291                                }
4292                                continue 'tool_rounds;
4293                            }
4294                        }
4295                        self.emit(RoderEvent::TurnFailed(TurnFailed {
4296                            thread_id: req.thread_id.clone(),
4297                            turn_id: turn_id.clone(),
4298                            error: error.clone(),
4299                            error_kind: None,
4300                            usage: None,
4301                            timestamp: OffsetDateTime::now_utc(),
4302                        }))
4303                        .await;
4304                        self.complete_team_member_turn_with_result(
4305                            &req.thread_id,
4306                            &turn_id,
4307                            TeamMemberStatus::Failed,
4308                            (!assistant_text.trim().is_empty()).then(|| assistant_text.clone()),
4309                            Some(error),
4310                        )
4311                        .await?;
4312                        return Err(err);
4313                    }
4314                };
4315
4316                let inference_timestamp = OffsetDateTime::now_utc();
4317                self.emit(RoderEvent::InferenceEventReceived(InferenceEventReceived {
4318                    thread_id: req.thread_id.clone(),
4319                    turn_id: turn_id.clone(),
4320                    event: event.clone(),
4321                    timestamp: inference_timestamp,
4322                }))
4323                .await;
4324
4325                match event {
4326                    InferenceEvent::MessageDelta(delta) => {
4327                        if let Some((team_id, member)) =
4328                            self.teams.member_for_thread(&req.thread_id).await
4329                        {
4330                            self.emit(RoderEvent::TeamMemberMessageDelta(TeamMemberMessageDelta {
4331                                team_id,
4332                                member_id: member.id,
4333                                member_thread_id: req.thread_id.clone(),
4334                                turn_id: turn_id.clone(),
4335                                delta: delta.text.clone(),
4336                                timestamp: OffsetDateTime::now_utc(),
4337                            }))
4338                            .await;
4339                        }
4340                        if is_final_answer_phase(delta.phase.as_deref()) {
4341                            assistant_text.push_str(&delta.text);
4342                        } else if let Some(last) = phase_messages.last_mut()
4343                            && last.phase == delta.phase
4344                        {
4345                            last.text.push_str(&delta.text);
4346                        } else {
4347                            phase_messages.push(AssistantMessage {
4348                                text: delta.text,
4349                                phase: delta.phase,
4350                            });
4351                        }
4352                    }
4353                    InferenceEvent::ReasoningDelta(delta) => reasoning_text.push_str(&delta.text),
4354                    InferenceEvent::ToolCallCompleted(call) => tool_calls.push(call),
4355                    InferenceEvent::Failed(failure) => {
4356                        speed_policy.record_failure();
4357                        let error = failure.message;
4358                        self.persist_turn_item(
4359                            &req.thread_id,
4360                            &turn_id,
4361                            &TranscriptItem::Error(ErrorRecord {
4362                                message: error.clone(),
4363                            }),
4364                        )
4365                        .await?;
4366                        self.emit(RoderEvent::TurnFailed(TurnFailed {
4367                            thread_id: req.thread_id.clone(),
4368                            turn_id: turn_id.clone(),
4369                            error: error.clone(),
4370                            error_kind: None,
4371                            usage: None,
4372                            timestamp: OffsetDateTime::now_utc(),
4373                        }))
4374                        .await;
4375                        self.complete_team_member_turn_with_result(
4376                            &req.thread_id,
4377                            &turn_id,
4378                            TeamMemberStatus::Failed,
4379                            (!assistant_text.trim().is_empty()).then(|| assistant_text.clone()),
4380                            Some(error),
4381                        )
4382                        .await?;
4383                        return Ok(TurnRunOutcome::Stopped);
4384                    }
4385                    InferenceEvent::Usage(usage) => {
4386                        turn_usage.add_assign(&usage);
4387                    }
4388                    InferenceEvent::Completed(metadata) => {
4389                        turn_finish_reason = metadata
4390                            .stop_reason
4391                            .as_deref()
4392                            .map(finish_reason_from_stop_reason);
4393                    }
4394                    InferenceEvent::Compaction(progress) => {
4395                        // Persist the boundary as soon as the provider emits a
4396                        // completed compaction item. ChatGPT Codex often aborts
4397                        // the SSE stream right after ("error decoding response
4398                        // body"), so waiting for ProviderMetadata loses the
4399                        // boundary and every subsequent request re-compacts.
4400                        if progress.status == "completed"
4401                            && let Some(item) = progress.item
4402                        {
4403                            let already = stream_compaction_item
4404                                .as_ref()
4405                                .and_then(|existing| {
4406                                    existing.get("id").and_then(serde_json::Value::as_str)
4407                                })
4408                                .zip(item.get("id").and_then(serde_json::Value::as_str))
4409                                .is_some_and(|(a, b)| a == b);
4410                            if !already {
4411                                stream_compaction_item = Some(item.clone());
4412                                let boundary = TranscriptItem::ProviderMetadata(
4413                                    crate::compaction::provider_metadata_from_compaction_item(item),
4414                                );
4415                                self.persist_turn_item(&req.thread_id, &turn_id, &boundary)
4416                                    .await?;
4417                                transcript.push(boundary);
4418                                transcript =
4419                                    crate::compaction::trim_to_last_compaction_boundary(transcript);
4420                                compacted_this_turn = true;
4421                            }
4422                        }
4423                    }
4424                    InferenceEvent::HostedToolCallStarted(_)
4425                    | InferenceEvent::HostedToolCallCompleted(_)
4426                    | InferenceEvent::ToolCallStarted(_)
4427                    | InferenceEvent::ToolCallDelta(_) => {}
4428                    InferenceEvent::ProviderMetadata(mut metadata) => {
4429                        if let Some(compaction_item) = stream_compaction_item.as_ref() {
4430                            let _ = crate::compaction::ensure_provider_metadata_compaction(
4431                                &mut metadata,
4432                                compaction_item,
4433                            );
4434                        }
4435                        provider_metadata = Some(metadata);
4436                    }
4437                }
4438            }
4439
4440            speed_policy.record_model_output(
4441                !assistant_text.is_empty() || !phase_messages.is_empty(),
4442                tool_calls.len(),
4443            );
4444            if tool_calls.is_empty() {
4445                let steers = self.drain_turn_steers(&turn_id).await;
4446                if !steers.is_empty() {
4447                    for message in phase_messages {
4448                        let item = TranscriptItem::AssistantMessage(message);
4449                        self.persist_turn_item(&req.thread_id, &turn_id, &item)
4450                            .await?;
4451                        transcript.push(item);
4452                        self.persist_model_profile_segment(
4453                            &req.thread_id,
4454                            &turn_id,
4455                            model_profile.as_ref(),
4456                            &provider,
4457                            &model,
4458                            "assistant",
4459                        )
4460                        .await?;
4461                    }
4462                    if !assistant_text.is_empty() {
4463                        let assistant = TranscriptItem::AssistantMessage(AssistantMessage {
4464                            text: assistant_text,
4465                            phase: Some(FINAL_ANSWER_PHASE.to_string()),
4466                        });
4467                        self.persist_turn_item(&req.thread_id, &turn_id, &assistant)
4468                            .await?;
4469                        transcript.push(assistant);
4470                        self.persist_model_profile_segment(
4471                            &req.thread_id,
4472                            &turn_id,
4473                            model_profile.as_ref(),
4474                            &provider,
4475                            &model,
4476                            "assistant",
4477                        )
4478                        .await?;
4479                    }
4480                    if let Some(metadata) = provider_metadata {
4481                        let had_provider_compaction =
4482                            crate::compaction::provider_metadata_has_compaction(&metadata);
4483                        let item = TranscriptItem::ProviderMetadata(metadata);
4484                        self.persist_turn_item(&req.thread_id, &turn_id, &item)
4485                            .await?;
4486                        transcript.push(item);
4487                        if had_provider_compaction {
4488                            // Server-side compaction replaced the prior window;
4489                            // drop pre-boundary items so the next request does
4490                            // not re-send (and re-compact) the full history.
4491                            transcript =
4492                                crate::compaction::trim_to_last_compaction_boundary(transcript);
4493                            compacted_this_turn = true;
4494                        }
4495                    }
4496                    self.append_steers(&req, &turn_id, &mut transcript, steers)
4497                        .await?;
4498                    continue;
4499                }
4500                if !deadline_finalization_requested
4501                    && req.task_ledger_required
4502                    && runtime_profile == RuntimeProfile::Eval
4503                    && task_ledger_completion_reminders < TASK_LEDGER_COMPLETION_REMINDER_LIMIT
4504                    && (!assistant_text.trim().is_empty() || !phase_messages.is_empty())
4505                    && let Some(prompt) = task_ledger_completion_prompt(&transcript)
4506                {
4507                    task_ledger_completion_reminders += 1;
4508                    let item = TranscriptItem::UserMessage(UserMessage::text(prompt));
4509                    self.persist_turn_item(&req.thread_id, &turn_id, &item)
4510                        .await?;
4511                    transcript.push(item);
4512                    continue;
4513                }
4514                if !deadline_finalization_requested
4515                    && let Some(prompt) = verification_gate.blocking_prompt()
4516                {
4517                    speed_policy.record_verification_required();
4518                    self.emit(RoderEvent::VerificationRequired(VerificationRequired {
4519                        thread_id: req.thread_id.clone(),
4520                        turn_id: turn_id.clone(),
4521                        reason: verification_gate.reason(),
4522                        changed_files: verification_gate.changed_files(),
4523                        tool_evidence: verification_gate.tool_evidence.clone(),
4524                        tests_run: verification_gate.tests_run.clone(),
4525                        open_gaps: verification_gate.open_gaps.clone(),
4526                        timestamp: OffsetDateTime::now_utc(),
4527                    }))
4528                    .await;
4529                    let item = TranscriptItem::UserMessage(UserMessage::text(prompt));
4530                    self.persist_turn_item(&req.thread_id, &turn_id, &item)
4531                        .await?;
4532                    transcript.push(item);
4533                    continue;
4534                }
4535                // Loop persistence nudge: in non-interactive/eval runs, when the
4536                // model returns a final message with no tool calls, gently prompt
4537                // it to keep going (bounded by `empty_tool_call_nudges`) before
4538                // ending the turn. Gated to non-interactive/eval so interactive
4539                // chat completions are never delayed, and only fires when the
4540                // model actually produced a final answer to re-examine.
4541                if !deadline_finalization_requested
4542                    && runtime_profile != RuntimeProfile::Interactive
4543                    && cfg.reliability.empty_tool_call_nudges > 0
4544                    && empty_tool_call_nudges_used < cfg.reliability.empty_tool_call_nudges
4545                    && (!assistant_text.trim().is_empty() || !phase_messages.is_empty())
4546                {
4547                    empty_tool_call_nudges_used += 1;
4548                    let item = TranscriptItem::UserMessage(UserMessage::text(
4549                        RELIABILITY_CONTINUATION_PROMPT.to_string(),
4550                    ));
4551                    self.persist_turn_item(&req.thread_id, &turn_id, &item)
4552                        .await?;
4553                    transcript.push(item);
4554                    continue;
4555                }
4556                if deadline_finalization_requested
4557                    && assistant_text.trim().is_empty()
4558                    && phase_messages.is_empty()
4559                {
4560                    assistant_text = format!(
4561                        "Deadline finalization completed without model text. {}",
4562                        turn_partial_result(&transcript)
4563                    );
4564                }
4565                final_phase_messages = phase_messages;
4566                final_assistant_text = assistant_text;
4567                final_reasoning_text = reasoning_text;
4568                final_provider_metadata = provider_metadata;
4569                exhausted_tool_rounds = false;
4570                break;
4571            }
4572
4573            for message in phase_messages {
4574                let item = TranscriptItem::AssistantMessage(message);
4575                self.persist_turn_item(&req.thread_id, &turn_id, &item)
4576                    .await?;
4577                transcript.push(item);
4578                self.persist_model_profile_segment(
4579                    &req.thread_id,
4580                    &turn_id,
4581                    model_profile.as_ref(),
4582                    &provider,
4583                    &model,
4584                    "assistant",
4585                )
4586                .await?;
4587            }
4588            if !assistant_text.is_empty() {
4589                transcript.push(TranscriptItem::AssistantMessage(AssistantMessage {
4590                    text: assistant_text,
4591                    phase: Some(FINAL_ANSWER_PHASE.to_string()),
4592                }));
4593                self.persist_model_profile_segment(
4594                    &req.thread_id,
4595                    &turn_id,
4596                    model_profile.as_ref(),
4597                    &provider,
4598                    &model,
4599                    "assistant",
4600                )
4601                .await?;
4602            }
4603            if let Some(metadata) = provider_metadata {
4604                let had_provider_compaction =
4605                    crate::compaction::provider_metadata_has_compaction(&metadata);
4606                let item = TranscriptItem::ProviderMetadata(metadata);
4607                self.persist_turn_item(&req.thread_id, &turn_id, &item)
4608                    .await?;
4609                transcript.push(item);
4610                if had_provider_compaction {
4611                    // Server-side compaction replaced the prior window; drop
4612                    // pre-boundary items so subsequent tool rounds use the
4613                    // compact window as the next input.
4614                    transcript = crate::compaction::trim_to_last_compaction_boundary(transcript);
4615                    compacted_this_turn = true;
4616                }
4617            }
4618            for call in &tool_calls {
4619                let tool_item = TranscriptItem::ToolCall(ToolCallRecord {
4620                    id: call.id.clone(),
4621                    name: call.name.clone(),
4622                    arguments: call.arguments.clone(),
4623                });
4624                self.persist_turn_item(&req.thread_id, &turn_id, &tool_item)
4625                    .await?;
4626                transcript.push(tool_item);
4627                self.persist_model_profile_segment(
4628                    &req.thread_id,
4629                    &turn_id,
4630                    model_profile.as_ref(),
4631                    &provider,
4632                    &model,
4633                    "tool_call",
4634                )
4635                .await?;
4636            }
4637            if let Some(deadline) = turn_deadline
4638                && deadline_expired(deadline)
4639            {
4640                self.fail_turn_due_to_deadline(&req.thread_id, &turn_id, deadline, &transcript)
4641                    .await?;
4642                return Ok(TurnRunOutcome::Stopped);
4643            }
4644            let results = self
4645                .route_tool_calls(
4646                    &req.thread_id,
4647                    &turn_id,
4648                    tool_calls,
4649                    parallel_tool_calls,
4650                    Some(workspace.as_str()),
4651                    turn_deadline,
4652                )
4653                .await?;
4654            let reliability_limit = reliability.record_tool_results(
4655                &cfg.reliability,
4656                &results,
4657                runtime_profile == RuntimeProfile::Interactive,
4658            );
4659            for result in results {
4660                verification_gate.record_tool_result(&result);
4661                transcript.push(TranscriptItem::ToolResult(result));
4662                self.persist_model_profile_segment(
4663                    &req.thread_id,
4664                    &turn_id,
4665                    model_profile.as_ref(),
4666                    &provider,
4667                    &model,
4668                    "tool_result",
4669                )
4670                .await?;
4671            }
4672            if let Some(limit) = reliability_limit {
4673                if limit.decision == ReliabilityLimitDecision::RequestContinuation {
4674                    // Loop persistence: rather than ending the turn, reset the
4675                    // consecutive-failure counter, nudge the model to keep going,
4676                    // and continue the round loop. Bounded by the per-turn tool
4677                    // failure ceiling, `max_model_calls_per_turn`, and
4678                    // `MAX_TOOL_ROUNDS_PER_TURN`.
4679                    self.record_reliability_limit_continuation(
4680                        &req.thread_id,
4681                        &turn_id,
4682                        &provider,
4683                        &model,
4684                        limit,
4685                    )
4686                    .await;
4687                    reliability.reset_consecutive_failures();
4688                    let nudge = TranscriptItem::UserMessage(UserMessage::text(
4689                        RELIABILITY_CONTINUATION_PROMPT.to_string(),
4690                    ));
4691                    self.persist_turn_item(&req.thread_id, &turn_id, &nudge)
4692                        .await?;
4693                    transcript.push(nudge);
4694                } else {
4695                    self.fail_turn_due_to_reliability_limit(
4696                        &req.thread_id,
4697                        &turn_id,
4698                        &provider,
4699                        &model,
4700                        limit,
4701                        &transcript,
4702                    )
4703                    .await?;
4704                    return Ok(TurnRunOutcome::Stopped);
4705                }
4706            }
4707            transcript = self
4708                .compact_transcript_if_needed(
4709                    &req.thread_id,
4710                    &turn_id,
4711                    &provider,
4712                    &model,
4713                    transcript,
4714                    self.compaction_options_for_turn(&req.thread_id, !compacted_this_turn),
4715                )
4716                .await?;
4717            compacted_this_turn = compacted_this_turn
4718                || transcript
4719                    .iter()
4720                    .any(crate::compaction::is_compaction_boundary);
4721        }
4722
4723        if exhausted_tool_rounds {
4724            let message =
4725                format!("tool call limit reached after {MAX_TOOL_ROUNDS_PER_TURN} rounds");
4726            self.persist_turn_item(
4727                &req.thread_id,
4728                &turn_id,
4729                &TranscriptItem::Error(ErrorRecord {
4730                    message: message.clone(),
4731                }),
4732            )
4733            .await?;
4734            self.emit(RoderEvent::TurnFailed(TurnFailed {
4735                thread_id: req.thread_id.clone(),
4736                turn_id: turn_id.clone(),
4737                error: message.clone(),
4738                error_kind: None,
4739                usage: None,
4740                timestamp: OffsetDateTime::now_utc(),
4741            }))
4742            .await;
4743            self.complete_team_member_turn_with_result(
4744                &req.thread_id,
4745                &turn_id,
4746                TeamMemberStatus::Failed,
4747                None,
4748                Some(message),
4749            )
4750            .await?;
4751            return Ok(TurnRunOutcome::Stopped);
4752        }
4753
4754        if !final_reasoning_text.is_empty() {
4755            self.persist_turn_item(
4756                &req.thread_id,
4757                &turn_id,
4758                &TranscriptItem::ReasoningSummary(ReasoningSummary {
4759                    text: final_reasoning_text,
4760                }),
4761            )
4762            .await?;
4763        }
4764        for message in final_phase_messages {
4765            self.persist_turn_item(
4766                &req.thread_id,
4767                &turn_id,
4768                &TranscriptItem::AssistantMessage(message),
4769            )
4770            .await?;
4771            self.persist_model_profile_segment(
4772                &req.thread_id,
4773                &turn_id,
4774                model_profile.as_ref(),
4775                &provider,
4776                &model,
4777                "assistant",
4778            )
4779            .await?;
4780        }
4781        if !final_assistant_text.is_empty() {
4782            self.persist_turn_item(
4783                &req.thread_id,
4784                &turn_id,
4785                &TranscriptItem::AssistantMessage(AssistantMessage {
4786                    text: final_assistant_text.clone(),
4787                    phase: Some(FINAL_ANSWER_PHASE.to_string()),
4788                }),
4789            )
4790            .await?;
4791            self.persist_model_profile_segment(
4792                &req.thread_id,
4793                &turn_id,
4794                model_profile.as_ref(),
4795                &provider,
4796                &model,
4797                "assistant",
4798            )
4799            .await?;
4800        }
4801        if let Some(metadata) = final_provider_metadata {
4802            self.persist_turn_item(
4803                &req.thread_id,
4804                &turn_id,
4805                &TranscriptItem::ProviderMetadata(metadata),
4806            )
4807            .await?;
4808        }
4809
4810        let turn_usage_tokens = turn_usage.total_tokens as i64;
4811        let completed_usage = (!turn_usage.is_empty()).then_some(turn_usage.clone());
4812        self.record_thread_usage_metadata(&req.thread_id, &turn_usage)
4813            .await?;
4814        self.goals
4815            .account_turn_usage(
4816                &req.thread_id,
4817                turn_usage_tokens,
4818                OffsetDateTime::now_utc() - turn_started_at,
4819            )
4820            .await?;
4821        let (cleanup, ownership) = self.await_provider_turn_cleanup(&turn_id).await;
4822        self.record_turn_lifecycle_with_ownership(
4823            req.thread_id.clone(),
4824            turn_id.clone(),
4825            TurnLifecycleState::Completed,
4826            cleanup,
4827            None,
4828            ownership,
4829        )
4830        .await;
4831        self.emit(RoderEvent::TurnCompleted(TurnCompleted {
4832            thread_id: req.thread_id.clone(),
4833            turn_id: turn_id.clone(),
4834            usage: completed_usage,
4835            finish_reason: turn_finish_reason,
4836            timestamp: OffsetDateTime::now_utc(),
4837        }))
4838        .await;
4839        self.complete_team_member_turn_with_result(
4840            &req.thread_id,
4841            &turn_id,
4842            TeamMemberStatus::Completed,
4843            (!final_assistant_text.is_empty()).then_some(final_assistant_text),
4844            None,
4845        )
4846        .await?;
4847        Ok(TurnRunOutcome::Completed)
4848    }
4849
4850    async fn drain_turn_steers(&self, turn_id: &TurnId) -> Vec<QueuedTurnSteer> {
4851        let Some(active) = self.active_turns.read().await.get(turn_id).cloned() else {
4852            return Vec::new();
4853        };
4854        let mut steers = active.steers.lock().await;
4855        std::mem::take(&mut *steers)
4856    }
4857
4858    async fn route_tool_calls(
4859        self: &Arc<Self>,
4860        thread_id: &ThreadId,
4861        turn_id: &TurnId,
4862        calls: Vec<ToolCallCompleted>,
4863        parallel: bool,
4864        workspace: Option<&str>,
4865        deadline: Option<OffsetDateTime>,
4866    ) -> anyhow::Result<Vec<ToolResultRecord>> {
4867        // Enforce the agent_swarm exclusivity rule (roadmap 104, Task 2):
4868        // `agent_swarm` must be the only tool call in a model response. A mixed
4869        // or multi-swarm batch is denied wholesale with actionable retry text so
4870        // the model re-issues `agent_swarm` by itself, and every tool_call_id
4871        // still gets a response (keeping the chat-completions transcript valid).
4872        if let Some(violation) = roder_api::subagents::agent_swarm_batch_violation(
4873            calls.iter().map(|call| call.name.as_str()),
4874        ) {
4875            let message = violation.deny_message();
4876            return Ok(calls
4877                .into_iter()
4878                .map(|call| ToolResultRecord {
4879                    id: call.id,
4880                    name: Some(call.name),
4881                    result: message.clone(),
4882                    display_payload: None,
4883                    is_error: true,
4884                })
4885                .collect());
4886        }
4887        // Emit swarm lifecycle events on the bus (roadmap 104, Task 1) so any
4888        // app-server/SDK/TUI client can observe a swarm as a whole; per-child
4889        // progress flows through the existing Subagent* trace events.
4890        let swarm_call = (calls.len() == 1
4891            && calls[0].name == roder_api::subagents::AGENT_SWARM_TOOL_NAME)
4892            .then(|| (calls[0].id.clone(), calls[0].arguments.clone()));
4893        if let Some((tool_id, args)) = &swarm_call {
4894            self.emit(RoderEvent::AgentSwarmStarted(
4895                roder_api::subagents::AgentSwarmStarted {
4896                    thread_id: thread_id.clone(),
4897                    turn_id: turn_id.clone(),
4898                    tool_id: tool_id.clone(),
4899                    child_count: agent_swarm_child_count(args),
4900                    timestamp: OffsetDateTime::now_utc(),
4901                },
4902            ))
4903            .await;
4904        }
4905
4906        let force_sequential = calls
4907            .iter()
4908            .any(|call| crate::agent_control_tools::is_agent_control_tool(&call.name));
4909        let results =
4910            if parallel && !force_sequential {
4911                try_join_all(calls.into_iter().map(|call| {
4912                    self.route_tool_call(thread_id, turn_id, call, workspace, deadline)
4913                }))
4914                .await
4915            } else {
4916                let mut results = Vec::with_capacity(calls.len());
4917                for call in calls {
4918                    results.push(
4919                        self.route_tool_call(thread_id, turn_id, call, workspace, deadline)
4920                            .await?,
4921                    );
4922                }
4923                Ok(results)
4924            }?;
4925
4926        if let Some((tool_id, _)) = &swarm_call
4927            && let Some(result) = results.iter().find(|result| &result.id == tool_id)
4928            && let Some((completed, failed, aborted)) = parse_swarm_counts(&result.result)
4929        {
4930            self.emit(RoderEvent::AgentSwarmCompleted(
4931                roder_api::subagents::AgentSwarmCompleted {
4932                    thread_id: thread_id.clone(),
4933                    turn_id: turn_id.clone(),
4934                    tool_id: tool_id.clone(),
4935                    completed,
4936                    failed,
4937                    aborted,
4938                    timestamp: OffsetDateTime::now_utc(),
4939                },
4940            ))
4941            .await;
4942        }
4943
4944        Ok(results)
4945    }
4946
4947    async fn fail_turn_with_error(
4948        &self,
4949        thread_id: &ThreadId,
4950        turn_id: &TurnId,
4951        message: String,
4952    ) -> anyhow::Result<()> {
4953        self.persist_turn_item(
4954            thread_id,
4955            turn_id,
4956            &TranscriptItem::Error(ErrorRecord {
4957                message: message.clone(),
4958            }),
4959        )
4960        .await?;
4961        self.emit(RoderEvent::TurnFailed(TurnFailed {
4962            thread_id: thread_id.clone(),
4963            turn_id: turn_id.clone(),
4964            error: message.clone(),
4965            error_kind: None,
4966            usage: None,
4967            timestamp: OffsetDateTime::now_utc(),
4968        }))
4969        .await;
4970        self.complete_team_member_turn_with_result(
4971            thread_id,
4972            turn_id,
4973            TeamMemberStatus::Failed,
4974            None,
4975            Some(message),
4976        )
4977        .await?;
4978        Ok(())
4979    }
4980
4981    async fn fail_turn_due_to_deadline(
4982        &self,
4983        thread_id: &ThreadId,
4984        turn_id: &TurnId,
4985        deadline: OffsetDateTime,
4986        transcript: &[TranscriptItem],
4987    ) -> anyhow::Result<()> {
4988        let partial_result = turn_partial_result(transcript);
4989        self.emit(RoderEvent::TurnPartialResult(TurnPartialResult {
4990            thread_id: thread_id.clone(),
4991            turn_id: turn_id.clone(),
4992            summary: partial_result.clone(),
4993            timestamp: OffsetDateTime::now_utc(),
4994        }))
4995        .await;
4996        self.emit(RoderEvent::TurnDeadlineExceeded(TurnDeadlineExceeded {
4997            thread_id: thread_id.clone(),
4998            turn_id: turn_id.clone(),
4999            deadline,
5000            partial_result: partial_result.clone(),
5001            timestamp: OffsetDateTime::now_utc(),
5002        }))
5003        .await;
5004        let message = "turn deadline expired".to_string();
5005        self.persist_turn_item(
5006            thread_id,
5007            turn_id,
5008            &TranscriptItem::Error(ErrorRecord {
5009                message: format!("{message}: {partial_result}"),
5010            }),
5011        )
5012        .await?;
5013        self.emit(RoderEvent::TurnFailed(TurnFailed {
5014            thread_id: thread_id.clone(),
5015            turn_id: turn_id.clone(),
5016            error: message.clone(),
5017            error_kind: Some("deadline_timeout".to_string()),
5018            usage: None,
5019            timestamp: OffsetDateTime::now_utc(),
5020        }))
5021        .await;
5022        self.complete_team_member_turn_with_result(
5023            thread_id,
5024            turn_id,
5025            TeamMemberStatus::Failed,
5026            None,
5027            Some(format!("{message}: {partial_result}")),
5028        )
5029        .await?;
5030        Ok(())
5031    }
5032
5033    async fn start_deadline_finalization(
5034        &self,
5035        thread_id: &ThreadId,
5036        turn_id: &TurnId,
5037        transcript: &mut Vec<TranscriptItem>,
5038        remaining_seconds: u64,
5039    ) -> anyhow::Result<()> {
5040        let item = TranscriptItem::UserMessage(crate::deadline_policy::finalization_message(
5041            remaining_seconds,
5042        ));
5043        self.persist_turn_item(thread_id, turn_id, &item).await?;
5044        transcript.push(item);
5045        self.emit(RoderEvent::TurnPartialResult(TurnPartialResult {
5046            thread_id: thread_id.clone(),
5047            turn_id: turn_id.clone(),
5048            summary: turn_partial_result(transcript),
5049            timestamp: OffsetDateTime::now_utc(),
5050        }))
5051        .await;
5052        Ok(())
5053    }
5054
5055    /// Emits the reliability-limit event for a continuation (loop persistence)
5056    /// without failing the turn. The caller resets the failure counter and
5057    /// injects a nudge so the round loop keeps going.
5058    async fn record_reliability_limit_continuation(
5059        &self,
5060        thread_id: &ThreadId,
5061        turn_id: &TurnId,
5062        provider: &str,
5063        model: &str,
5064        limit: ReliabilityLimitHit,
5065    ) {
5066        self.emit(RoderEvent::ReliabilityLimitRecorded(
5067            ReliabilityLimitRecorded {
5068                context: ReliabilityContext {
5069                    thread_id: thread_id.clone(),
5070                    turn_id: turn_id.clone(),
5071                    tool_id: None,
5072                    tool_name: None,
5073                    provider: Some(provider.to_string()),
5074                    model: Some(model.to_string()),
5075                },
5076                error_class: limit.error_class,
5077                limit_kind: limit.limit_kind,
5078                decision: limit.decision,
5079                current: limit.current,
5080                limit: limit.limit,
5081                details: ReliabilityDetails::redacted(&limit.message),
5082                timestamp: OffsetDateTime::now_utc(),
5083            },
5084        ))
5085        .await;
5086    }
5087
5088    async fn fail_turn_due_to_reliability_limit(
5089        &self,
5090        thread_id: &ThreadId,
5091        turn_id: &TurnId,
5092        provider: &str,
5093        model: &str,
5094        limit: ReliabilityLimitHit,
5095        transcript: &[TranscriptItem],
5096    ) -> anyhow::Result<()> {
5097        self.emit(RoderEvent::ReliabilityLimitRecorded(
5098            ReliabilityLimitRecorded {
5099                context: ReliabilityContext {
5100                    thread_id: thread_id.clone(),
5101                    turn_id: turn_id.clone(),
5102                    tool_id: None,
5103                    tool_name: None,
5104                    provider: Some(provider.to_string()),
5105                    model: Some(model.to_string()),
5106                },
5107                error_class: limit.error_class,
5108                limit_kind: limit.limit_kind,
5109                decision: limit.decision,
5110                current: limit.current,
5111                limit: limit.limit,
5112                details: ReliabilityDetails::redacted(&limit.message),
5113                timestamp: OffsetDateTime::now_utc(),
5114            },
5115        ))
5116        .await;
5117        let partial_result = turn_partial_result(transcript);
5118        self.emit(RoderEvent::TurnPartialResult(TurnPartialResult {
5119            thread_id: thread_id.clone(),
5120            turn_id: turn_id.clone(),
5121            summary: partial_result.clone(),
5122            timestamp: OffsetDateTime::now_utc(),
5123        }))
5124        .await;
5125        let message = format!("reliability limit reached: {}", limit.message);
5126        self.persist_turn_item(
5127            thread_id,
5128            turn_id,
5129            &TranscriptItem::Error(ErrorRecord {
5130                message: format!("{message}: {partial_result}"),
5131            }),
5132        )
5133        .await?;
5134        self.emit(RoderEvent::TurnFailed(TurnFailed {
5135            thread_id: thread_id.clone(),
5136            turn_id: turn_id.clone(),
5137            error: message.clone(),
5138            error_kind: Some("reliability_limit".to_string()),
5139            usage: None,
5140            timestamp: OffsetDateTime::now_utc(),
5141        }))
5142        .await;
5143        self.complete_team_member_turn_with_result(
5144            thread_id,
5145            turn_id,
5146            TeamMemberStatus::Failed,
5147            None,
5148            Some(format!("{message}: {partial_result}")),
5149        )
5150        .await?;
5151        Ok(())
5152    }
5153
5154    async fn append_steers(
5155        &self,
5156        req: &StartTurnRequest,
5157        turn_id: &TurnId,
5158        transcript: &mut Vec<TranscriptItem>,
5159        steers: Vec<QueuedTurnSteer>,
5160    ) -> anyhow::Result<()> {
5161        for queued in steers {
5162            let mut steer = queued.message;
5163            steer.text = steer.text.trim().to_string();
5164            if steer.text.is_empty() && steer.images.is_empty() {
5165                continue;
5166            }
5167            let item = TranscriptItem::UserMessage(steer);
5168            self.persist_turn_item(&req.thread_id, turn_id, &item)
5169                .await?;
5170            transcript.push(item);
5171            if let Some(ack) = queued.mailbox_ack {
5172                self.teams
5173                    .mark_mailbox_messages_delivered(&ack.team_id, turn_id, &ack.message_ids)
5174                    .await?;
5175            }
5176        }
5177        Ok(())
5178    }
5179
5180    async fn persist_model_profile_segment(
5181        &self,
5182        thread_id: &ThreadId,
5183        turn_id: &TurnId,
5184        profile: Option<&ModelHarnessProfile>,
5185        provider: &str,
5186        model: &str,
5187        segment: &str,
5188    ) -> anyhow::Result<()> {
5189        let item = TranscriptItem::ProviderMetadata(model_profile_segment_metadata(
5190            profile, provider, model, segment,
5191        ));
5192        self.persist_turn_item(thread_id, turn_id, &item).await
5193    }
5194
5195    /**
5196     * Allowlists apply to built-in tools only; external tools are advertised with their
5197     * host-supplied schemas as given. An external tool shadows a built-in with the same name in
5198     * both advertisement and dispatch (see `route_tool_call`).
5199     */
5200    fn filtered_tool_specs(
5201        &self,
5202        cfg: &RuntimeConfig,
5203        model: &str,
5204        profile: Option<&ModelHarnessProfile>,
5205        thread_allowlist: &[String],
5206        external_tools: &[roder_api::tools::ToolSpec],
5207    ) -> Vec<roder_api::tools::ToolSpec> {
5208        let mut specs = self
5209            .tool_registry
5210            .specs_for_edit_tool_with_schema_policy(
5211                edit_tool_for_model(cfg, model),
5212                schema_policy_for_model(profile),
5213            )
5214            .into_iter()
5215            .filter(|spec| {
5216                allowlist_permits(&cfg.tool_allowlist, &spec.name)
5217                    && allowlist_permits(thread_allowlist, &spec.name)
5218                    && !external_tools.iter().any(|tool| tool.name == spec.name)
5219            })
5220            .collect::<Vec<_>>();
5221        specs.extend(external_tools.iter().cloned());
5222        specs
5223    }
5224
5225    fn task_ledger_tool_specs(
5226        &self,
5227        profile: Option<&ModelHarnessProfile>,
5228    ) -> Vec<roder_api::tools::ToolSpec> {
5229        self.tool_registry
5230            .get(TASK_LEDGER_TOOL_NAME)
5231            .map(|tool| {
5232                tool.spec()
5233                    .normalized_for_model_profile(schema_policy_for_model(profile))
5234            })
5235            .into_iter()
5236            .collect()
5237    }
5238
5239    pub(crate) fn engine_for(&self, provider: &str) -> anyhow::Result<Arc<dyn InferenceEngine>> {
5240        self.registry
5241            .inference_engine(provider)
5242            .or_else(|| {
5243                self.registry
5244                    .default_inference_engine()
5245                    .filter(|engine| provider.is_empty() || engine.id() == provider)
5246            })
5247            .ok_or_else(|| anyhow::anyhow!("inference provider {provider:?} is not registered"))
5248    }
5249
5250    pub async fn emit(&self, event: RoderEvent) -> EventEnvelope {
5251        if let Some(record) = Self::lifecycle_record_for_event(&event) {
5252            let _ = self.persist_turn_lifecycle_record(&record).await;
5253        }
5254        let envelope = self.bus.emit(event);
5255        if let (Some(store), Some(thread_id)) = (&self.thread_store, envelope.thread_id.as_ref())
5256            && should_persist_thread_event(thread_id)
5257        {
5258            let _ = store.append_event(thread_id, &envelope).await;
5259        }
5260        // Registered event sinks (e.g. process extensions) receive the
5261        // persisted envelope through bounded per-sink queues; a slow sink
5262        // never blocks emit or turn progress.
5263        let dispatcher = self
5264            .event_sink_dispatcher
5265            .get_or_init(|| async {
5266                crate::event_sink_dispatch::EventSinkDispatcher::start(
5267                    &self.registry.event_sinks,
5268                    self.bus.clone(),
5269                )
5270            })
5271            .await;
5272        if !dispatcher.is_empty() {
5273            dispatcher.dispatch(&envelope, &self.bus);
5274        }
5275        envelope
5276    }
5277
5278    /// Records a projected thread item event and persists it through the configured thread store.
5279    ///
5280    /// App-server protocol notification bridges currently call this after translating runtime
5281    /// events into item events. Headless runtime consumers that subscribe to `RoderEvent` directly
5282    /// must make the same call if they need the derived item event stream persisted.
5283    pub async fn record_thread_item_event_kind(
5284        &self,
5285        thread_id: &ThreadId,
5286        turn_id: &TurnId,
5287        timestamp: OffsetDateTime,
5288        kind: ThreadItemEventKind,
5289    ) -> anyhow::Result<ThreadItemEvent> {
5290        let seq = self.next_thread_item_event_seq(thread_id).await?;
5291        let item_event = ThreadItemEvent {
5292            seq,
5293            event_id: format!("{turn_id}-item-event-{seq}"),
5294            thread_id: thread_id.clone(),
5295            turn_id: turn_id.clone(),
5296            timestamp,
5297            event: kind,
5298        };
5299        if let Some(store) = &self.thread_store {
5300            store.append_item_event(thread_id, &item_event).await?;
5301        }
5302        self.remember_thread_item_event(&item_event).await?;
5303        Ok(item_event)
5304    }
5305
5306    async fn next_thread_item_event_seq(&self, thread_id: &ThreadId) -> anyhow::Result<u64> {
5307        self.ensure_thread_item_cache(thread_id).await?;
5308        Ok(self
5309            .thread_item_cache
5310            .lock()
5311            .await
5312            .next_item_event_seq(thread_id))
5313    }
5314
5315    pub async fn thread_item_exists(
5316        &self,
5317        thread_id: &ThreadId,
5318        turn_id: &TurnId,
5319        item_id: &str,
5320    ) -> anyhow::Result<bool> {
5321        self.ensure_thread_item_cache(thread_id).await?;
5322        Ok(self
5323            .thread_item_cache
5324            .lock()
5325            .await
5326            .thread_item_exists(thread_id, turn_id, item_id))
5327    }
5328
5329    pub async fn current_reasoning_item_id(
5330        &self,
5331        thread_id: &ThreadId,
5332        turn_id: &TurnId,
5333    ) -> anyhow::Result<Option<String>> {
5334        self.ensure_thread_item_cache(thread_id).await?;
5335        Ok(self
5336            .thread_item_cache
5337            .lock()
5338            .await
5339            .current_reasoning_item_id(thread_id, turn_id))
5340    }
5341
5342    async fn remember_thread_item_event(&self, item_event: &ThreadItemEvent) -> anyhow::Result<()> {
5343        self.ensure_thread_item_cache(&item_event.thread_id).await?;
5344        self.thread_item_cache
5345            .lock()
5346            .await
5347            .remember_item_event(item_event);
5348        Ok(())
5349    }
5350
5351    pub async fn latest_transcript_item_index(
5352        &self,
5353        thread_id: &ThreadId,
5354        turn_id: &TurnId,
5355    ) -> anyhow::Result<Option<usize>> {
5356        self.ensure_thread_item_cache(thread_id).await?;
5357        Ok(self
5358            .thread_item_cache
5359            .lock()
5360            .await
5361            .latest_transcript_item_index(thread_id, turn_id))
5362    }
5363
5364    async fn next_transcript_item_index(
5365        &self,
5366        thread_id: &ThreadId,
5367        turn_id: &TurnId,
5368    ) -> anyhow::Result<usize> {
5369        self.ensure_thread_item_cache(thread_id).await?;
5370        Ok(self
5371            .thread_item_cache
5372            .lock()
5373            .await
5374            .next_transcript_item_index(thread_id, turn_id))
5375    }
5376
5377    async fn remember_transcript_item_index(
5378        &self,
5379        thread_id: &ThreadId,
5380        turn_id: &TurnId,
5381        item_index: usize,
5382    ) -> anyhow::Result<()> {
5383        self.ensure_thread_item_cache(thread_id).await?;
5384        self.thread_item_cache
5385            .lock()
5386            .await
5387            .remember_transcript_item_index(thread_id, turn_id, item_index);
5388        Ok(())
5389    }
5390
5391    async fn ensure_thread_item_cache(&self, thread_id: &ThreadId) -> anyhow::Result<()> {
5392        if self
5393            .thread_item_cache
5394            .lock()
5395            .await
5396            .contains_thread(thread_id)
5397        {
5398            return Ok(());
5399        }
5400
5401        let snapshot = if let Some(store) = &self.thread_store {
5402            store.load_thread(thread_id).await?
5403        } else {
5404            None
5405        };
5406        self.thread_item_cache.lock().await.ensure_thread(
5407            thread_id,
5408            ThreadItemCacheEntry::from_snapshot(snapshot.as_ref()),
5409        );
5410        Ok(())
5411    }
5412
5413    pub(crate) async fn persist_turn_item(
5414        &self,
5415        thread_id: &ThreadId,
5416        turn_id: &TurnId,
5417        item: &TranscriptItem,
5418    ) -> anyhow::Result<()> {
5419        let item_index = self.next_transcript_item_index(thread_id, turn_id).await?;
5420        let timestamp = OffsetDateTime::now_utc();
5421        self.emit(RoderEvent::TranscriptItemAppended(TranscriptItemAppended {
5422            thread_id: thread_id.clone(),
5423            turn_id: turn_id.clone(),
5424            item_type: match item {
5425                TranscriptItem::UserMessage(_) => "user_message",
5426                TranscriptItem::AssistantMessage(_) => "assistant_message",
5427                TranscriptItem::ReasoningSummary(_) => "reasoning_summary",
5428                TranscriptItem::ToolCall(_) => "tool_call",
5429                TranscriptItem::ToolResult(_) => "tool_result",
5430                TranscriptItem::FileChange(_) => "file_change",
5431                TranscriptItem::ContextCompaction(_) => "context_compaction",
5432                TranscriptItem::Error(_) => "error",
5433                TranscriptItem::ProviderMetadata(_) => "provider_metadata",
5434            }
5435            .to_string(),
5436            item_index: Some(item_index),
5437            item: Some(item.clone()),
5438            timestamp,
5439        }))
5440        .await;
5441        self.remember_transcript_item_index(thread_id, turn_id, item_index)
5442            .await?;
5443        Ok(())
5444    }
5445}
5446
5447fn transcript_has_images(transcript: &[TranscriptItem]) -> bool {
5448    transcript.iter().any(|item| {
5449        matches!(
5450            item,
5451            TranscriptItem::UserMessage(message) if !message.images.is_empty()
5452        )
5453    })
5454}
5455
5456fn transcript_has_task_ledger(transcript: &[TranscriptItem]) -> bool {
5457    transcript.iter().any(|item| {
5458        matches!(
5459            item,
5460            TranscriptItem::ToolResult(result)
5461                if result.name.as_deref() == Some(TASK_LEDGER_TOOL_NAME) && !result.is_error
5462        )
5463    })
5464}
5465
5466fn task_ledger_completion_prompt(transcript: &[TranscriptItem]) -> Option<String> {
5467    let latest = transcript.iter().rev().find_map(|item| match item {
5468        TranscriptItem::ToolResult(result)
5469            if result.name.as_deref() == Some(TASK_LEDGER_TOOL_NAME) && !result.is_error =>
5470        {
5471            Some(result.result.as_str())
5472        }
5473        _ => None,
5474    })?;
5475    if !task_ledger_has_open_items(latest) {
5476        return None;
5477    }
5478
5479    let mut ledger = latest.chars().take(1500).collect::<String>();
5480    if latest.chars().nth(1500).is_some() {
5481        ledger.push_str("...");
5482    }
5483    Some(format!(
5484        "Task Ledger Completion Required: the latest task ledger still has pending or in-progress items. Do not provide a final answer yet. Use tools to complete the remaining scoreable work, create or update any required output files, then call `{TASK_LEDGER_TOOL_NAME}` with every task completed and evidence before finalizing.\n\nLatest ledger:\n{ledger}"
5485    ))
5486}
5487
5488fn task_ledger_deadline_completion_prompt(
5489    remaining_seconds: u64,
5490    reserve_seconds: u64,
5491    completion_prompt: &str,
5492) -> String {
5493    format!(
5494        "Eval deadline scoreable completion: {remaining_seconds} seconds remain in the {reserve_seconds}-second finalization reserve. Do not browse, search, or start slow work. Use the available tools now to create or update the required scoreable output files, run only a quick local check if needed, then update the task ledger to completed before finalizing.\n\n{completion_prompt}"
5495    )
5496}
5497
5498fn task_ledger_scoreable_checkpoint_prompt(
5499    remaining_seconds: u64,
5500    completion_prompt: &str,
5501) -> String {
5502    format!(
5503        "Scoreable Output Checkpoint: {remaining_seconds} seconds remain before the eval deadline. Before any further research, browsing, or long commands, use tools now to ensure the required output file(s) exist with the best evidence-backed answer, even if provisional. If a scoreable file already exists, read it and preserve that candidate unless you have stronger task-specific evidence for a replacement. Do not overwrite a plausible dated, historical, or local-evidence candidate with a current live-page, partial-coverage, or weaker guess merely to refresh the checkpoint. You may continue refining afterward, but do not apologize or finalize until the scoreable file exists and the task ledger is updated.\n\n{completion_prompt}"
5504    )
5505}
5506
5507fn task_ledger_has_open_items(ledger: &str) -> bool {
5508    ledger.lines().any(|line| {
5509        let line = line.trim_start();
5510        line.starts_with("- pending:") || line.starts_with("- in_progress:")
5511    })
5512}
5513
5514fn turn_deadline_for_config(cfg: &RuntimeConfig) -> Option<OffsetDateTime> {
5515    cfg.turn_deadline_seconds
5516        .filter(|seconds| *seconds > 0)
5517        .map(|seconds| OffsetDateTime::now_utc() + Duration::seconds(seconds as i64))
5518}
5519
5520fn deadline_expired(deadline: OffsetDateTime) -> bool {
5521    OffsetDateTime::now_utc() >= deadline
5522}
5523
5524pub(crate) fn deadline_remaining_seconds(deadline: Option<OffsetDateTime>) -> Option<u64> {
5525    let deadline = deadline?;
5526    if deadline <= OffsetDateTime::now_utc() {
5527        return Some(0);
5528    }
5529    Some(
5530        (deadline - OffsetDateTime::now_utc())
5531            .unsigned_abs()
5532            .as_secs()
5533            .max(1),
5534    )
5535}
5536
5537fn deadline_instant(deadline: OffsetDateTime) -> tokio::time::Instant {
5538    let now = OffsetDateTime::now_utc();
5539    if deadline <= now {
5540        return tokio::time::Instant::now();
5541    }
5542    tokio::time::Instant::now() + (deadline - now).unsigned_abs()
5543}
5544
5545fn inference_timeout_deadline(
5546    deadline: Option<OffsetDateTime>,
5547    runtime_profile: RuntimeProfile,
5548    task_ledger_required: bool,
5549    reserve_seconds: u64,
5550    finalization_requested: bool,
5551    task_ledger_scoreable_checkpoints: u8,
5552    transcript: &[TranscriptItem],
5553) -> Option<(OffsetDateTime, InferenceTimeoutAction)> {
5554    let deadline = deadline?;
5555    if runtime_profile == RuntimeProfile::Eval
5556        && task_ledger_required
5557        && !finalization_requested
5558        && task_ledger_scoreable_checkpoints < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
5559        && TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS > reserve_seconds
5560        && task_ledger_completion_prompt(transcript).is_some()
5561    {
5562        let checkpoint_deadline =
5563            deadline - Duration::seconds(TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS as i64);
5564        if checkpoint_deadline > OffsetDateTime::now_utc() {
5565            return Some((
5566                checkpoint_deadline,
5567                InferenceTimeoutAction::ScoreableCheckpoint,
5568            ));
5569        }
5570    }
5571    if runtime_profile == RuntimeProfile::Eval && !finalization_requested {
5572        return Some((
5573            deadline - Duration::seconds(reserve_seconds as i64),
5574            InferenceTimeoutAction::Finalization,
5575        ));
5576    }
5577    Some((deadline, InferenceTimeoutAction::Finalization))
5578}
5579
5580fn turn_partial_result(transcript: &[TranscriptItem]) -> String {
5581    let tool_results = transcript
5582        .iter()
5583        .filter(|item| matches!(item, TranscriptItem::ToolResult(_)))
5584        .count();
5585    let assistant_messages = transcript
5586        .iter()
5587        .filter(|item| matches!(item, TranscriptItem::AssistantMessage(_)))
5588        .count();
5589    format!(
5590        "partial turn state: {} transcript items, {assistant_messages} assistant messages, {tool_results} tool results",
5591        transcript.len()
5592    )
5593}
5594
5595fn reasoning_for_model(cfg: &RuntimeConfig, model: &str) -> ReasoningConfig {
5596    let level = effective_reasoning_for_model(cfg, model);
5597    match level.as_str() {
5598        "" | REASONING_NONE => ReasoningConfig::default(),
5599        level => ReasoningConfig {
5600            enabled: true,
5601            level: Some(level.to_string()),
5602        },
5603    }
5604}
5605
5606fn server_side_compaction_threshold(cfg: &RuntimeConfig, model: &str) -> Option<u32> {
5607    let entry = lookup_model(model)?;
5608    if !entry.supports_compaction {
5609        return None;
5610    }
5611    cfg.auto_compact_token_limit
5612        .or_else(|| {
5613            model_profile_for_model(cfg, model).and_then(|profile| profile.auto_compact_token_limit)
5614        })
5615        .or(Some(entry.auto_compact_token_limit))
5616        .filter(|threshold| *threshold > 0)
5617}
5618
5619pub(crate) fn tool_search_for_provider_model(
5620    cfg: &RuntimeConfig,
5621    provider: &str,
5622    model: &str,
5623) -> ToolSearchConfig {
5624    let mut resolved = cfg.tool_search.clone();
5625    if let Some(provider_config) = cfg.provider_tool_search.get(provider) {
5626        provider_config.apply_to(&mut resolved);
5627    }
5628    if let Some(model_config) = cfg.model_tool_search.get(model) {
5629        model_config.apply_to(&mut resolved);
5630    }
5631    resolved
5632}
5633
5634fn parallel_tool_calls_for_model(cfg: &RuntimeConfig, model: &str) -> bool {
5635    cfg.model_parallel_tool_calls
5636        .get(model)
5637        .copied()
5638        .or_else(|| {
5639            model_profile_for_model(cfg, model).and_then(|profile| profile.parallel_tool_calls)
5640        })
5641        .unwrap_or(true)
5642}
5643
5644/// Count the children an `agent_swarm` call will launch from its arguments
5645/// (item-based spawns plus resumes). Lenient: returns 0 on malformed input.
5646fn agent_swarm_child_count(arguments: &str) -> usize {
5647    serde_json::from_str::<roder_api::subagents::AgentSwarmRequest>(arguments)
5648        .map(|request| request.items.len() + request.resume_agent_ids.len())
5649        .unwrap_or(0)
5650}
5651
5652/// Parse `completed`/`failed`/`aborted` counts from an `<agent_swarm_result>`
5653/// summary line, e.g. `<summary>completed: 2, failed: 1</summary>`. Omitted
5654/// buckets default to 0. Returns `None` when the text is not a swarm result.
5655fn parse_swarm_counts(text: &str) -> Option<(usize, usize, usize)> {
5656    if !text.contains("<agent_swarm_result>") {
5657        return None;
5658    }
5659    let bucket = |label: &str| -> usize {
5660        let needle = format!("{label}: ");
5661        text.find(&needle)
5662            .map(|start| start + needle.len())
5663            .map(|start| {
5664                text[start..]
5665                    .chars()
5666                    .take_while(|c| c.is_ascii_digit())
5667                    .collect::<String>()
5668            })
5669            .and_then(|digits| digits.parse().ok())
5670            .unwrap_or(0)
5671    };
5672    Some((bucket("completed"), bucket("failed"), bucket("aborted")))
5673}
5674
5675fn effective_reasoning_for_model(cfg: &RuntimeConfig, model: &str) -> String {
5676    let base_reasoning = default_effective_reasoning_for_model(cfg, model);
5677    if cfg.dynamic_workflows.effort_profile == DynamicWorkflowEffortProfile::Ultracode {
5678        return ultracode_reasoning_level_for_model(
5679            model,
5680            &cfg.speed_policy.ultracode_reasoning,
5681            &base_reasoning,
5682        );
5683    }
5684    base_reasoning
5685}
5686
5687fn default_effective_reasoning_for_model(cfg: &RuntimeConfig, model: &str) -> String {
5688    let Some(entry) = lookup_model(model) else {
5689        return cfg
5690            .reasoning
5691            .clone()
5692            .unwrap_or_else(|| REASONING_NONE.to_string());
5693    };
5694    if entry.supported_reasoning.is_empty() {
5695        return REASONING_NONE.to_string();
5696    }
5697    cfg.reasoning
5698        .as_deref()
5699        .filter(|reasoning| {
5700            entry
5701                .supported_reasoning
5702                .iter()
5703                .any(|option| option.effort == *reasoning)
5704        })
5705        .map(str::to_string)
5706        .or_else(|| {
5707            model_profile_for_model(cfg, model)
5708                .and_then(|profile| profile.reasoning.orientation)
5709                .filter(|reasoning| {
5710                    entry
5711                        .supported_reasoning
5712                        .iter()
5713                        .any(|option| option.effort == reasoning)
5714                })
5715        })
5716        .unwrap_or_else(|| entry.default_reasoning.to_string())
5717}
5718
5719fn validate_reasoning_effort(model: &str, effort: &str) -> anyhow::Result<()> {
5720    if effort == REASONING_NONE && !model_supports_reasoning(model, effort) {
5721        return Ok(());
5722    }
5723    let Some(entry) = lookup_model(model) else {
5724        return Ok(());
5725    };
5726    if entry
5727        .supported_reasoning
5728        .iter()
5729        .any(|option| option.effort == effort)
5730    {
5731        Ok(())
5732    } else {
5733        anyhow::bail!("model {model} does not support reasoning effort {effort}")
5734    }
5735}
5736
5737fn validate_runtime_config_reasoning(cfg: &RuntimeConfig) -> anyhow::Result<()> {
5738    let Some(reasoning) = cfg.reasoning.as_deref() else {
5739        return Ok(());
5740    };
5741    let Some(entry) = lookup_model(&cfg.default_model) else {
5742        return Ok(());
5743    };
5744    if entry.provider != PROVIDER_GEMINI {
5745        return Ok(());
5746    }
5747    validate_reasoning_effort(&cfg.default_model, reasoning)
5748}
5749
5750fn validate_runtime_inference_router_config(
5751    registry: &ExtensionRegistry,
5752    cfg: &RuntimeConfig,
5753) -> anyhow::Result<()> {
5754    if !cfg.inference_router.enabled {
5755        return Ok(());
5756    }
5757    let Some(router_id) = cfg.inference_router.router_id.as_deref() else {
5758        anyhow::bail!("inference_router.enabled requires inference_router.router");
5759    };
5760    if registry.inference_router(router_id).is_some() {
5761        return Ok(());
5762    }
5763    let available = registry
5764        .inference_routers
5765        .iter()
5766        .map(|router| router.id())
5767        .collect::<Vec<_>>()
5768        .join(", ");
5769    if available.is_empty() {
5770        anyhow::bail!("inference router {router_id:?} is not registered");
5771    }
5772    anyhow::bail!(
5773        "inference router {router_id:?} is not registered; available routers: {available}"
5774    );
5775}
5776
5777fn model_supports_reasoning(model: &str, effort: &str) -> bool {
5778    lookup_model(model)
5779        .map(|entry| {
5780            entry
5781                .supported_reasoning
5782                .iter()
5783                .any(|option| option.effort == effort)
5784        })
5785        .unwrap_or(false)
5786}
5787
5788fn is_final_answer_phase(phase: Option<&str>) -> bool {
5789    phase.is_none_or(|phase| phase.is_empty() || phase == FINAL_ANSWER_PHASE)
5790}
5791
5792fn edit_tool_for_model<'a>(cfg: &'a RuntimeConfig, model: &'a str) -> Option<&'a str> {
5793    cfg.model_edit_tools
5794        .get(model)
5795        .map(String::as_str)
5796        .or_else(|| {
5797            cfg.model_profiles
5798                .get(model)
5799                .and_then(|profile| profile.edit_tool.as_deref())
5800        })
5801        .or_else(|| lookup_model(model).and_then(|entry| entry.edit_tool))
5802        .or(Some(EDIT_TOOL_EDIT))
5803}
5804
5805fn model_profile_for_model(cfg: &RuntimeConfig, model: &str) -> Option<ModelHarnessProfile> {
5806    cfg.model_profiles
5807        .get(model)
5808        .cloned()
5809        .or_else(|| built_in_model_profile(model))
5810}
5811
5812pub(crate) fn allowlist_permits(allowlist: &[String], tool_name: &str) -> bool {
5813    allowlist.is_empty() || allowlist.iter().any(|allowed| allowed == tool_name)
5814}
5815
5816/// Provider-aware profile resolution for the active turn.
5817///
5818/// Many model ids are shared across providers (for example Cursor proxies
5819/// `claude-opus-4-8`). Resolving the harness profile by id alone picks the
5820/// first catalog entry, which assigns cross-provider ids the wrong family and
5821/// instruction overlay (e.g. a `cursor/claude-opus-4-8` turn would otherwise
5822/// inherit the Anthropic overlay). Prefer the explicit `(provider, model)`
5823/// catalog entry, keeping user-configured profile overrides as the top
5824/// precedence.
5825fn model_profile_for_provider_model(
5826    cfg: &RuntimeConfig,
5827    provider: &str,
5828    model: &str,
5829) -> Option<ModelHarnessProfile> {
5830    cfg.model_profiles
5831        .get(model)
5832        .cloned()
5833        .or_else(|| built_in_model_profile_for_provider(provider, model))
5834}
5835
5836fn schema_policy_for_model(profile: Option<&ModelHarnessProfile>) -> ModelSchemaPolicy {
5837    profile
5838        .map(|profile| profile.schema_policy)
5839        .unwrap_or_default()
5840}
5841
5842fn model_profile_segment_metadata(
5843    profile: Option<&ModelHarnessProfile>,
5844    provider: &str,
5845    model: &str,
5846    segment: &str,
5847) -> serde_json::Value {
5848    serde_json::json!({
5849        "kind": MODEL_PROFILE_TRACE_KIND,
5850        "segment": segment,
5851        "provider": provider,
5852        "model": model,
5853        "profileModel": profile.map(|profile| profile.model.as_str()).unwrap_or(model),
5854        "providerFamily": profile.map(|profile| profile.provider_family),
5855        "editTool": profile.and_then(|profile| profile.edit_tool.as_deref()),
5856        "schemaPolicy": profile.map(|profile| profile.schema_policy),
5857        "instructionOverlay": profile.map(|profile| profile.instruction_overlay),
5858        "parallelToolCalls": profile.and_then(|profile| profile.parallel_tool_calls),
5859        "autoCompactTokenLimit": profile.and_then(|profile| profile.auto_compact_token_limit),
5860    })
5861}
5862
5863fn model_switch_summary(
5864    transcript: &[TranscriptItem],
5865    profile: Option<&ModelHarnessProfile>,
5866    provider: &str,
5867    model: &str,
5868    tools: &[roder_api::tools::ToolSpec],
5869) -> Option<String> {
5870    let previous = latest_model_profile_segment(transcript)?;
5871    let previous_model = previous
5872        .get("model")
5873        .and_then(serde_json::Value::as_str)
5874        .unwrap_or_default();
5875    let previous_provider = previous
5876        .get("provider")
5877        .and_then(serde_json::Value::as_str)
5878        .unwrap_or_default();
5879    if previous_model == model && previous_provider == provider {
5880        return None;
5881    }
5882
5883    let previous_profile = previous
5884        .get("profileModel")
5885        .and_then(serde_json::Value::as_str)
5886        .unwrap_or(previous_model);
5887    let current_profile = profile
5888        .map(|profile| profile.model.as_str())
5889        .unwrap_or(model);
5890    let previous_edit_tool = previous
5891        .get("editTool")
5892        .and_then(serde_json::Value::as_str)
5893        .unwrap_or("none");
5894    let current_edit_tool = profile
5895        .and_then(|profile| profile.edit_tool.as_deref())
5896        .unwrap_or("none");
5897    let tool_names = tools
5898        .iter()
5899        .map(|tool| tool.name.as_str())
5900        .take(12)
5901        .collect::<Vec<_>>()
5902        .join(", ");
5903    Some(format!(
5904        "{MODEL_SWITCH_SUMMARY_PREFIX} previous profile {previous_provider}/{previous_profile} used edit tool {previous_edit_tool}. Current profile {provider}/{current_profile} uses edit tool {current_edit_tool}. Available tools now: {}.",
5905        if tool_names.is_empty() {
5906            "none"
5907        } else {
5908            &tool_names
5909        }
5910    ))
5911}
5912
5913fn latest_model_profile_segment(transcript: &[TranscriptItem]) -> Option<&serde_json::Value> {
5914    transcript.iter().rev().find_map(|item| {
5915        let TranscriptItem::ProviderMetadata(value) = item else {
5916            return None;
5917        };
5918        (value.get("kind").and_then(serde_json::Value::as_str) == Some(MODEL_PROFILE_TRACE_KIND))
5919            .then_some(value)
5920    })
5921}
5922
5923pub fn validate_edit_tool(value: &str) -> anyhow::Result<()> {
5924    match value.trim() {
5925        EDIT_TOOL_PATCH | EDIT_TOOL_EDIT => Ok(()),
5926        _ => anyhow::bail!(
5927            "unsupported edit_tool {value:?}; allowed values: {EDIT_TOOL_PATCH}, {EDIT_TOOL_EDIT}"
5928        ),
5929    }
5930}
5931
5932fn should_persist_thread_event(thread_id: &str) -> bool {
5933    !is_synthetic_event_thread_id(thread_id)
5934}
5935
5936#[cfg(test)]
5937#[path = "runtime/codex_v2_tests.rs"]
5938mod codex_v2_tests;
5939
5940#[cfg(test)]
5941#[path = "runtime/codex_v2_lifecycle_tests.rs"]
5942mod codex_v2_lifecycle_tests;
5943
5944#[cfg(test)]
5945mod tests {
5946    use super::*;
5947    use futures::stream;
5948    use roder_api::catalog::{
5949        PROVIDER_MOCK, REASONING_HIGH, REASONING_LOW, REASONING_MEDIUM, REASONING_MINIMAL,
5950        REASONING_NONE, REASONING_XHIGH,
5951    };
5952    use roder_api::extension::ExtensionRegistryBuilder;
5953    use roder_api::inference::{
5954        CompletionMetadata, InferenceCapabilities, InferenceEngine, InferenceEventStream,
5955        InferenceProviderContext, InferenceTurnContext, MessageDelta, ModelDescriptor,
5956        ModelInstructionOverlay, ModelProfileReasoning, ModelSchemaPolicy, ProviderFamily,
5957        ReasoningEffortDescriptor,
5958    };
5959    use roder_api::inference_routing::{
5960        InferenceRouter, InferenceRoutingContext, InferenceRoutingDecision, InferenceRoutingOutcome,
5961    };
5962    use roder_api::thread::ThreadStoreFactory;
5963    use roder_api::tools::{ToolContributor, ToolExecutor, ToolSpec};
5964    use roder_ext_jsonl_thread_store::store::JsonlThreadStoreFactory;
5965    use std::sync::Mutex as StdMutex;
5966
5967    fn test_workspace() -> String {
5968        std::env::current_dir().unwrap().display().to_string()
5969    }
5970
5971    #[test]
5972    fn interactive_explicit_turn_deadline_has_bounded_remaining_time() {
5973        let config = RuntimeConfig {
5974            runtime_profile: RuntimeProfile::Interactive,
5975            turn_deadline_seconds: Some(60),
5976            ..RuntimeConfig::default()
5977        };
5978
5979        let deadline = turn_deadline_for_config(&config).expect("configured turn deadline");
5980        let remaining = deadline_remaining_seconds(Some(deadline));
5981
5982        assert!(
5983            matches!(remaining, Some(1..=60)),
5984            "interactive deadline must be positive and no longer than configured: {remaining:?}"
5985        );
5986    }
5987
5988    #[test]
5989    fn interactive_without_turn_deadline_stays_unbounded() {
5990        let config = RuntimeConfig {
5991            runtime_profile: RuntimeProfile::Interactive,
5992            turn_deadline_seconds: None,
5993            ..RuntimeConfig::default()
5994        };
5995
5996        assert_eq!(turn_deadline_for_config(&config), None);
5997    }
5998
5999    struct MetadataMissingStore;
6000
6001    #[async_trait::async_trait]
6002    impl ThreadStore for MetadataMissingStore {
6003        fn id(&self) -> roder_api::thread::ThreadStoreId {
6004            "metadata-missing-store".to_string()
6005        }
6006
6007        async fn create_thread(&self, metadata: ThreadMetadata) -> anyhow::Result<ThreadMetadata> {
6008            Ok(metadata)
6009        }
6010
6011        async fn list_threads(&self) -> anyhow::Result<Vec<ThreadMetadata>> {
6012            Ok(Vec::new())
6013        }
6014
6015        async fn load_thread(
6016            &self,
6017            _thread_id: &ThreadId,
6018        ) -> anyhow::Result<Option<ThreadSnapshot>> {
6019            Ok(Some(ThreadSnapshot {
6020                metadata: None,
6021                ..ThreadSnapshot::default()
6022            }))
6023        }
6024
6025        async fn append_event(
6026            &self,
6027            _thread_id: &ThreadId,
6028            _envelope: &EventEnvelope,
6029        ) -> anyhow::Result<()> {
6030            Ok(())
6031        }
6032    }
6033
6034    struct MetadataMissingStoreFactory;
6035
6036    impl ThreadStoreFactory for MetadataMissingStoreFactory {
6037        fn id(&self) -> roder_api::thread::ThreadStoreId {
6038            "metadata-missing-store".to_string()
6039        }
6040
6041        fn create(&self) -> Arc<dyn ThreadStore> {
6042            Arc::new(MetadataMissingStore)
6043        }
6044    }
6045
6046    #[test]
6047    fn synthetic_app_server_events_are_not_thread_events() {
6048        for thread_id in ["app-server", "runtime", "thread-workflow"] {
6049            assert!(!should_persist_thread_event(thread_id));
6050        }
6051        assert!(should_persist_thread_event("thread-discovery"));
6052        assert!(should_persist_thread_event("thread-plan"));
6053        assert!(should_persist_thread_event("thread-process"));
6054        assert!(should_persist_thread_event("thread-1"));
6055    }
6056
6057    #[test]
6058    fn server_side_compaction_uses_catalog_ninety_percent_default() {
6059        assert_eq!(
6060            server_side_compaction_threshold(&RuntimeConfig::default(), "gpt-5.5"),
6061            Some(945_000)
6062        );
6063        assert_eq!(
6064            server_side_compaction_threshold(&RuntimeConfig::default(), "gpt-5.3-codex-spark"),
6065            Some(115_200)
6066        );
6067    }
6068
6069    #[test]
6070    fn server_side_compaction_respects_explicit_config_override() {
6071        let cfg = RuntimeConfig {
6072            auto_compact_token_limit: Some(123_456),
6073            ..RuntimeConfig::default()
6074        };
6075
6076        assert_eq!(
6077            server_side_compaction_threshold(&cfg, "gpt-5.5"),
6078            Some(123_456)
6079        );
6080    }
6081
6082    #[tokio::test]
6083    async fn pre_request_compaction_runs_when_server_side_model_is_at_context_window() {
6084        let captured = Arc::new(StdMutex::new(None));
6085        let mut builder = ExtensionRegistryBuilder::new();
6086        builder.inference_engine(Arc::new(CapturingEngine {
6087            request: captured.clone(),
6088        }));
6089        let thread_root = std::env::temp_dir().join(format!(
6090            "roder-pre-request-compaction-{}",
6091            uuid::Uuid::new_v4()
6092        ));
6093        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
6094            base_path: thread_root.clone(),
6095        }));
6096        let runtime = Arc::new(
6097            Runtime::new(
6098                builder.build().unwrap(),
6099                RuntimeConfig {
6100                    default_provider: PROVIDER_MOCK.to_string(),
6101                    default_model: "gpt-5.5".to_string(),
6102                    file_backed_dynamic_context: true,
6103                    ..RuntimeConfig::default()
6104                },
6105            )
6106            .unwrap(),
6107        );
6108        let thread_id = runtime
6109            .create_thread(Some("Pre-request compaction".to_string()))
6110            .await
6111            .unwrap()
6112            .thread_id;
6113        let old_turn = "old-turn".to_string();
6114        runtime
6115            .persist_turn_item(
6116                &thread_id,
6117                &old_turn,
6118                &TranscriptItem::UserMessage(UserMessage::text("old context ".repeat(4_300_000))),
6119            )
6120            .await
6121            .unwrap();
6122
6123        let mut events = runtime.subscribe_events();
6124        runtime
6125            .start_turn(StartTurnRequest {
6126                thread_id: thread_id.clone(),
6127                message: "continue".to_string(),
6128                images: Vec::new(),
6129                provider_override: None,
6130                model_override: None,
6131                reasoning_override: None,
6132                workspace: test_workspace(),
6133                instructions: InstructionBundle::default(),
6134                developer_context: None,
6135                task_ledger_required: false,
6136            })
6137            .await
6138            .unwrap();
6139        loop {
6140            let envelope = tokio::time::timeout(std::time::Duration::from_secs(5), events.recv())
6141                .await
6142                .unwrap()
6143                .unwrap();
6144            if envelope.thread_id.as_deref() == Some(&thread_id)
6145                && matches!(envelope.event, RoderEvent::TurnCompleted(_))
6146            {
6147                break;
6148            }
6149        }
6150
6151        let request = captured.lock().unwrap().clone().unwrap();
6152        assert!(
6153            matches!(
6154                request.transcript.first(),
6155                Some(TranscriptItem::ContextCompaction(_))
6156            ),
6157            "provider request should start with a local emergency compaction item"
6158        );
6159        assert!(
6160            request.transcript.len() < 4,
6161            "provider request should not replay the full oversized prior transcript: {:?}",
6162            request.transcript
6163        );
6164
6165        let _ = std::fs::remove_dir_all(thread_root);
6166    }
6167
6168    #[tokio::test]
6169    async fn continue_after_context_window_failure_compacts_before_provider_request() {
6170        let captured = Arc::new(StdMutex::new(None));
6171        let mut builder = ExtensionRegistryBuilder::new();
6172        builder.inference_engine(Arc::new(CapturingEngine {
6173            request: captured.clone(),
6174        }));
6175        let thread_root = std::env::temp_dir().join(format!(
6176            "roder-context-failure-continue-{}",
6177            uuid::Uuid::new_v4()
6178        ));
6179        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
6180            base_path: thread_root.clone(),
6181        }));
6182        let runtime = Arc::new(
6183            Runtime::new(
6184                builder.build().unwrap(),
6185                RuntimeConfig {
6186                    default_provider: PROVIDER_MOCK.to_string(),
6187                    default_model: "gpt-5.5".to_string(),
6188                    file_backed_dynamic_context: true,
6189                    ..RuntimeConfig::default()
6190                },
6191            )
6192            .unwrap(),
6193        );
6194        let thread_id = runtime
6195            .create_thread(Some("Context failure continue".to_string()))
6196            .await
6197            .unwrap()
6198            .thread_id;
6199        let failed_turn = "failed-turn".to_string();
6200        runtime
6201            .persist_turn_item(
6202                &thread_id,
6203                &failed_turn,
6204                &TranscriptItem::UserMessage(UserMessage::text("old work ".repeat(10_000))),
6205            )
6206            .await
6207            .unwrap();
6208        runtime
6209            .persist_turn_item(
6210                &thread_id,
6211                &failed_turn,
6212                &TranscriptItem::Error(ErrorRecord {
6213                    message: "Your input exceeds the context window of this model. Please adjust your input and try again."
6214                        .to_string(),
6215                }),
6216            )
6217            .await
6218            .unwrap();
6219
6220        let mut events = runtime.subscribe_events();
6221        runtime
6222            .start_turn(StartTurnRequest {
6223                thread_id: thread_id.clone(),
6224                message: "continue".to_string(),
6225                images: Vec::new(),
6226                provider_override: None,
6227                model_override: None,
6228                reasoning_override: None,
6229                workspace: test_workspace(),
6230                instructions: InstructionBundle::default(),
6231                developer_context: None,
6232                task_ledger_required: false,
6233            })
6234            .await
6235            .unwrap();
6236        loop {
6237            let envelope = tokio::time::timeout(std::time::Duration::from_secs(5), events.recv())
6238                .await
6239                .unwrap()
6240                .unwrap();
6241            if envelope.thread_id.as_deref() == Some(&thread_id)
6242                && matches!(envelope.event, RoderEvent::TurnCompleted(_))
6243            {
6244                break;
6245            }
6246        }
6247
6248        let request = captured.lock().unwrap().clone().unwrap();
6249        assert!(
6250            matches!(
6251                request.transcript.first(),
6252                Some(TranscriptItem::ContextCompaction(_))
6253            ),
6254            "provider request after context-window failure should start with local compaction"
6255        );
6256        assert!(
6257            request
6258                .transcript
6259                .iter()
6260                .any(|item| matches!(item, TranscriptItem::UserMessage(message) if message.text == "continue")),
6261            "current continue prompt must be preserved: {:?}",
6262            request.transcript
6263        );
6264        assert!(
6265            !request.transcript.iter().any(
6266                |item| matches!(item, TranscriptItem::Error(error) if error.message.contains("context window"))
6267            ),
6268            "raw prior context-window error should be summarized, not replayed: {:?}",
6269            request.transcript
6270        );
6271
6272        let _ = std::fs::remove_dir_all(thread_root);
6273    }
6274
6275    #[tokio::test]
6276    async fn workspace_for_thread_falls_back_when_metadata_is_missing() {
6277        let workspace = test_workspace();
6278        let mut builder = ExtensionRegistryBuilder::new();
6279        builder.inference_engine(Arc::new(FakeInferenceEngine));
6280        builder.thread_store_factory(Arc::new(MetadataMissingStoreFactory));
6281        let runtime = Runtime::new(
6282            builder.build().unwrap(),
6283            RuntimeConfig {
6284                workspace: Some(workspace.clone()),
6285                ..RuntimeConfig::default()
6286            },
6287        )
6288        .unwrap();
6289
6290        let resolved = runtime
6291            .workspace_for_thread(&ThreadId::from("thread-workflow"))
6292            .await
6293            .unwrap();
6294
6295        assert_eq!(resolved, workspace);
6296    }
6297
6298    #[tokio::test]
6299    async fn automations_can_create_project_thread_with_model_overrides() {
6300        let runtime = Runtime::fake().unwrap();
6301        let workspace = std::env::temp_dir().join("project");
6302        let metadata = runtime
6303            .create_thread_with(CreateThreadRequest {
6304                title: Some("Automation: nightly status".to_string()),
6305                workspace: workspace.display().to_string(),
6306                workspace_id: None,
6307                root_id: None,
6308                provider: Some("mock".to_string()),
6309                model: Some("mock".to_string()),
6310                selection_mode: None,
6311                tool_allowlist: Vec::new(),
6312                developer_instructions: None,
6313                external_tools: Vec::new(),
6314                runner: None,
6315            })
6316            .await
6317            .unwrap();
6318
6319        assert_eq!(
6320            metadata.title.as_deref(),
6321            Some("Automation: nightly status")
6322        );
6323        assert_eq!(metadata.workspace, workspace.display().to_string());
6324        assert_eq!(metadata.provider.as_deref(), Some("mock"));
6325        assert_eq!(metadata.model.as_deref(), Some("mock"));
6326    }
6327
6328    #[tokio::test]
6329    async fn prior_provider_compaction_boundary_is_used_on_next_turn() {
6330        let captured = Arc::new(StdMutex::new(None));
6331        let mut builder = ExtensionRegistryBuilder::new();
6332        builder.inference_engine(Arc::new(CapturingEngine {
6333            request: captured.clone(),
6334        }));
6335        let thread_root = std::env::temp_dir().join(format!(
6336            "roder-provider-compaction-boundary-{}",
6337            uuid::Uuid::new_v4()
6338        ));
6339        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
6340            base_path: thread_root.clone(),
6341        }));
6342        let runtime = Arc::new(
6343            Runtime::new(
6344                builder.build().unwrap(),
6345                RuntimeConfig {
6346                    default_provider: PROVIDER_MOCK.to_string(),
6347                    default_model: "gpt-5.5".to_string(),
6348                    ..RuntimeConfig::default()
6349                },
6350            )
6351            .unwrap(),
6352        );
6353        let thread_id = runtime
6354            .create_thread(Some("Provider compaction boundary".to_string()))
6355            .await
6356            .unwrap()
6357            .thread_id;
6358        let old_turn = "old-turn".to_string();
6359        runtime
6360            .persist_turn_item(
6361                &thread_id,
6362                &old_turn,
6363                &TranscriptItem::UserMessage(UserMessage::text(
6364                    "old history that must not be replayed after provider compaction",
6365                )),
6366            )
6367            .await
6368            .unwrap();
6369        runtime
6370            .persist_turn_item(
6371                &thread_id,
6372                &old_turn,
6373                &TranscriptItem::AssistantMessage(AssistantMessage {
6374                    text: "old answer".to_string(),
6375                    phase: None,
6376                }),
6377            )
6378            .await
6379            .unwrap();
6380        runtime
6381            .persist_turn_item(
6382                &thread_id,
6383                &old_turn,
6384                &TranscriptItem::ProviderMetadata(serde_json::json!({
6385                    "output": [{
6386                        "id": "cmp_1",
6387                        "type": "compaction",
6388                        "encrypted_content": "opaque-state"
6389                    }]
6390                })),
6391            )
6392            .await
6393            .unwrap();
6394        runtime
6395            .persist_turn_item(
6396                &thread_id,
6397                &old_turn,
6398                &TranscriptItem::AssistantMessage(AssistantMessage {
6399                    text: "after compact".to_string(),
6400                    phase: None,
6401                }),
6402            )
6403            .await
6404            .unwrap();
6405
6406        let mut events = runtime.subscribe_events();
6407        runtime
6408            .start_turn(StartTurnRequest {
6409                thread_id: thread_id.clone(),
6410                message: "continue".to_string(),
6411                images: Vec::new(),
6412                provider_override: None,
6413                model_override: None,
6414                reasoning_override: None,
6415                workspace: test_workspace(),
6416                instructions: InstructionBundle::default(),
6417                developer_context: None,
6418                task_ledger_required: false,
6419            })
6420            .await
6421            .unwrap();
6422        loop {
6423            let envelope = tokio::time::timeout(std::time::Duration::from_secs(5), events.recv())
6424                .await
6425                .unwrap()
6426                .unwrap();
6427            if envelope.thread_id.as_deref() == Some(&thread_id)
6428                && matches!(envelope.event, RoderEvent::TurnCompleted(_))
6429            {
6430                break;
6431            }
6432        }
6433
6434        let request = captured.lock().unwrap().clone().unwrap();
6435        let compaction_idx = request.transcript.iter().position(|item| {
6436            matches!(
6437                item,
6438                TranscriptItem::ProviderMetadata(metadata)
6439                    if crate::compaction::provider_metadata_has_compaction(metadata)
6440            )
6441        });
6442        assert!(
6443            compaction_idx.is_some(),
6444            "next provider request should include the provider compaction item: {:?}",
6445            request.transcript
6446        );
6447        // Skill/context injectors may prepend non-history items, but no prior-turn
6448        // conversation may appear before the latest provider compaction boundary.
6449        let history_before_boundary =
6450            request
6451                .transcript
6452                .iter()
6453                .take(compaction_idx.unwrap())
6454                .any(|item| match item {
6455                    TranscriptItem::AssistantMessage(_)
6456                    | TranscriptItem::ToolCall(_)
6457                    | TranscriptItem::ToolResult(_) => true,
6458                    TranscriptItem::UserMessage(message) => {
6459                        !message.text.contains("<skills>") && message.text != "continue"
6460                    }
6461                    _ => false,
6462                });
6463        assert!(
6464            !history_before_boundary,
6465            "pre-provider-compaction conversation must not be replayed: {:?}",
6466            request.transcript
6467        );
6468        assert!(
6469            !request.transcript.iter().any(|item| {
6470                matches!(
6471                    item,
6472                    TranscriptItem::UserMessage(message)
6473                        if message.text.contains("old history that must not be replayed")
6474                )
6475            }),
6476            "pre-provider-compaction history must not be replayed: {:?}",
6477            request.transcript
6478        );
6479        assert!(
6480            request.transcript.iter().any(|item| {
6481                matches!(
6482                    item,
6483                    TranscriptItem::UserMessage(message) if message.text == "continue"
6484                )
6485            }),
6486            "current continue prompt must be preserved: {:?}",
6487            request.transcript
6488        );
6489        assert_eq!(
6490            request.runtime.auto_compact_token_limit,
6491            Some(945_000),
6492            "gpt-5.5 should keep server-side auto compaction configured"
6493        );
6494
6495        let _ = std::fs::remove_dir_all(thread_root);
6496    }
6497
6498    #[test]
6499    fn server_side_compaction_is_only_enabled_for_supported_models() {
6500        let cfg = RuntimeConfig {
6501            auto_compact_token_limit: Some(123_456),
6502            ..RuntimeConfig::default()
6503        };
6504
6505        assert_eq!(server_side_compaction_threshold(&cfg, "mock"), None);
6506        assert_eq!(
6507            server_side_compaction_threshold(&cfg, "codex-auto-review"),
6508            None
6509        );
6510    }
6511
6512    #[test]
6513    fn reasoning_is_disabled_for_models_without_reasoning_support() {
6514        let cfg = RuntimeConfig {
6515            reasoning: Some(REASONING_HIGH.to_string()),
6516            ..RuntimeConfig::default()
6517        };
6518
6519        assert_eq!(
6520            effective_reasoning_for_model(&cfg, "claude-haiku-4-5-20251001"),
6521            REASONING_NONE
6522        );
6523        assert_eq!(
6524            reasoning_for_model(&cfg, "claude-haiku-4-5-20251001"),
6525            ReasoningConfig::default()
6526        );
6527    }
6528
6529    #[test]
6530    fn unsupported_configured_reasoning_falls_back_to_model_default() {
6531        let cfg = RuntimeConfig {
6532            reasoning: Some(REASONING_MINIMAL.to_string()),
6533            ..RuntimeConfig::default()
6534        };
6535
6536        assert_eq!(
6537            effective_reasoning_for_model(&cfg, "gpt-5.5"),
6538            REASONING_MEDIUM
6539        );
6540    }
6541
6542    #[test]
6543    fn unsupported_configured_gemini_reasoning_is_rejected() {
6544        let mut builder = ExtensionRegistryBuilder::new();
6545        builder.inference_engine(std::sync::Arc::new(FakeInferenceEngine));
6546
6547        let err = match Runtime::new(
6548            builder.build().unwrap(),
6549            RuntimeConfig {
6550                default_model: "gemini-3.5-flash".to_string(),
6551                reasoning: Some(REASONING_XHIGH.to_string()),
6552                ..RuntimeConfig::default()
6553            },
6554        ) {
6555            Ok(_) => panic!("expected unsupported Gemini reasoning to be rejected"),
6556            Err(err) => err,
6557        };
6558
6559        assert!(
6560            err.to_string()
6561                .contains("model gemini-3.5-flash does not support reasoning effort xhigh")
6562        );
6563    }
6564
6565    #[tokio::test]
6566    async fn selecting_none_for_non_reasoning_model_preserves_stored_preference() {
6567        let runtime = Runtime::new(
6568            Runtime::fake().unwrap().registry,
6569            RuntimeConfig {
6570                reasoning: Some(REASONING_HIGH.to_string()),
6571                ..RuntimeConfig::default()
6572            },
6573        )
6574        .unwrap();
6575
6576        let cfg = runtime
6577            .select_provider(
6578                roder_api::catalog::PROVIDER_MOCK.to_string(),
6579                Some("claude-haiku-4-5-20251001".to_string()),
6580                Some(REASONING_NONE.to_string()),
6581            )
6582            .await
6583            .unwrap();
6584
6585        assert_eq!(cfg.reasoning.as_deref(), Some(REASONING_HIGH));
6586        assert_eq!(runtime.effective_reasoning().await, REASONING_NONE);
6587    }
6588
6589    #[tokio::test]
6590    async fn selecting_none_for_model_that_supports_none_updates_preference() {
6591        let runtime = Runtime::new(
6592            Runtime::fake().unwrap().registry,
6593            RuntimeConfig {
6594                reasoning: Some(REASONING_HIGH.to_string()),
6595                ..RuntimeConfig::default()
6596            },
6597        )
6598        .unwrap();
6599
6600        let cfg = runtime
6601            .select_provider(
6602                roder_api::catalog::PROVIDER_MOCK.to_string(),
6603                Some("mock".to_string()),
6604                Some(REASONING_NONE.to_string()),
6605            )
6606            .await
6607            .unwrap();
6608
6609        assert_eq!(cfg.reasoning.as_deref(), Some(REASONING_NONE));
6610    }
6611
6612    #[test]
6613    fn parallel_tool_calls_default_on_with_model_override() {
6614        assert!(parallel_tool_calls_for_model(
6615            &RuntimeConfig::default(),
6616            "custom-model"
6617        ));
6618
6619        let cfg = RuntimeConfig {
6620            model_parallel_tool_calls: std::collections::HashMap::from([(
6621                "custom-model".to_string(),
6622                false,
6623            )]),
6624            ..RuntimeConfig::default()
6625        };
6626
6627        assert!(!parallel_tool_calls_for_model(&cfg, "custom-model"));
6628        assert!(parallel_tool_calls_for_model(&cfg, "other-model"));
6629    }
6630
6631    #[test]
6632    fn profile_parallel_tool_calls_applies_between_config_and_default() {
6633        let cfg = RuntimeConfig {
6634            model_profiles: std::collections::HashMap::from([(
6635                "gpt-5.5".to_string(),
6636                test_model_profile("gpt-5.5"),
6637            )]),
6638            ..RuntimeConfig::default()
6639        };
6640
6641        assert!(!parallel_tool_calls_for_model(&cfg, "gpt-5.5"));
6642
6643        let cfg = RuntimeConfig {
6644            model_parallel_tool_calls: std::collections::HashMap::from([(
6645                "gpt-5.5".to_string(),
6646                true,
6647            )]),
6648            ..cfg
6649        };
6650
6651        assert!(parallel_tool_calls_for_model(&cfg, "gpt-5.5"));
6652    }
6653
6654    struct CapturingEngine {
6655        request: Arc<StdMutex<Option<AgentInferenceRequest>>>,
6656    }
6657
6658    #[async_trait::async_trait]
6659    impl InferenceEngine for CapturingEngine {
6660        fn id(&self) -> String {
6661            roder_api::catalog::PROVIDER_MOCK.to_string()
6662        }
6663
6664        fn capabilities(&self) -> InferenceCapabilities {
6665            InferenceCapabilities::coding_agent_default()
6666        }
6667
6668        async fn list_models(
6669            &self,
6670            _ctx: InferenceProviderContext<'_>,
6671        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
6672            Ok(roder_api::catalog::models_for_provider(
6673                roder_api::catalog::PROVIDER_MOCK,
6674                true,
6675            ))
6676        }
6677
6678        async fn stream_turn(
6679            &self,
6680            _ctx: InferenceTurnContext<'_>,
6681            request: AgentInferenceRequest,
6682        ) -> anyhow::Result<InferenceEventStream> {
6683            *self.request.lock().unwrap() = Some(request);
6684            Ok(Box::pin(stream::iter(vec![
6685                Ok(InferenceEvent::MessageDelta(MessageDelta {
6686                    text: "done".to_string(),
6687                    phase: None,
6688                })),
6689                Ok(InferenceEvent::Completed(CompletionMetadata {
6690                    stop_reason: Some("stop".to_string()),
6691                    provider_response_id: None,
6692                })),
6693            ])))
6694        }
6695    }
6696
6697    struct RoutingCaptureEngine {
6698        id: &'static str,
6699        models: Vec<ModelDescriptor>,
6700        requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
6701    }
6702
6703    #[async_trait::async_trait]
6704    impl InferenceEngine for RoutingCaptureEngine {
6705        fn id(&self) -> String {
6706            self.id.to_string()
6707        }
6708
6709        fn capabilities(&self) -> InferenceCapabilities {
6710            InferenceCapabilities::coding_agent_default()
6711        }
6712
6713        async fn list_models(
6714            &self,
6715            _ctx: InferenceProviderContext<'_>,
6716        ) -> anyhow::Result<Vec<ModelDescriptor>> {
6717            Ok(self.models.clone())
6718        }
6719
6720        async fn stream_turn(
6721            &self,
6722            _ctx: InferenceTurnContext<'_>,
6723            request: AgentInferenceRequest,
6724        ) -> anyhow::Result<InferenceEventStream> {
6725            self.requests.lock().unwrap().push(request);
6726            Ok(Box::pin(stream::iter(vec![
6727                Ok(InferenceEvent::MessageDelta(MessageDelta {
6728                    text: "routed".to_string(),
6729                    phase: None,
6730                })),
6731                Ok(InferenceEvent::Completed(CompletionMetadata {
6732                    stop_reason: Some("stop".to_string()),
6733                    provider_response_id: None,
6734                })),
6735            ])))
6736        }
6737    }
6738
6739    struct StaticRouter {
6740        id: &'static str,
6741        decision: InferenceRoutingDecision,
6742        contexts: Arc<StdMutex<Vec<InferenceRoutingContext>>>,
6743    }
6744
6745    #[async_trait::async_trait]
6746    impl InferenceRouter for StaticRouter {
6747        fn id(&self) -> String {
6748            self.id.to_string()
6749        }
6750
6751        async fn route(
6752            &self,
6753            context: InferenceRoutingContext,
6754        ) -> anyhow::Result<InferenceRoutingDecision> {
6755            self.contexts.lock().unwrap().push(context);
6756            Ok(self.decision.clone())
6757        }
6758    }
6759
6760    fn routing_test_model(id: &str, supported_reasoning: &[&str]) -> ModelDescriptor {
6761        ModelDescriptor {
6762            id: id.to_string(),
6763            name: id.to_string(),
6764            context_window: Some(128_000),
6765            default_reasoning: supported_reasoning
6766                .first()
6767                .map(|effort| (*effort).to_string()),
6768            supported_reasoning: supported_reasoning
6769                .iter()
6770                .map(|effort| ReasoningEffortDescriptor {
6771                    effort: (*effort).to_string(),
6772                    description: format!("{effort} reasoning"),
6773                })
6774                .collect(),
6775        }
6776    }
6777
6778    struct TaskLedgerCompletionGateEngine {
6779        calls: StdMutex<u32>,
6780        requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
6781    }
6782
6783    #[async_trait::async_trait]
6784    impl InferenceEngine for TaskLedgerCompletionGateEngine {
6785        fn id(&self) -> String {
6786            roder_api::catalog::PROVIDER_MOCK.to_string()
6787        }
6788
6789        fn capabilities(&self) -> InferenceCapabilities {
6790            InferenceCapabilities::coding_agent_default()
6791        }
6792
6793        async fn list_models(
6794            &self,
6795            _ctx: InferenceProviderContext<'_>,
6796        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
6797            Ok(roder_api::catalog::models_for_provider(
6798                roder_api::catalog::PROVIDER_MOCK,
6799                true,
6800            ))
6801        }
6802
6803        async fn stream_turn(
6804            &self,
6805            _ctx: InferenceTurnContext<'_>,
6806            request: AgentInferenceRequest,
6807        ) -> anyhow::Result<InferenceEventStream> {
6808            self.requests.lock().unwrap().push(request);
6809            let mut calls = self.calls.lock().unwrap();
6810            *calls += 1;
6811            let events = match *calls {
6812                1 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
6813                    id: "ledger-open".to_string(),
6814                    name: TASK_LEDGER_TOOL_NAME.to_string(),
6815                    arguments: serde_json::json!({
6816                        "tasks": [
6817                            {
6818                                "id": "inspect",
6819                                "content": "Inspect local assets",
6820                                "status": "completed",
6821                                "evidence": "listed workspace"
6822                            },
6823                            {
6824                                "id": "write",
6825                                "content": "Write /app/result.txt",
6826                                "status": "pending"
6827                            }
6828                        ],
6829                        "requireCompletionEvidence": true
6830                    })
6831                    .to_string(),
6832                }))],
6833                3 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
6834                    id: "ledger-complete".to_string(),
6835                    name: TASK_LEDGER_TOOL_NAME.to_string(),
6836                    arguments: serde_json::json!({
6837                        "tasks": [
6838                            {
6839                                "id": "inspect",
6840                                "content": "Inspect local assets",
6841                                "status": "completed",
6842                                "evidence": "listed workspace"
6843                            },
6844                            {
6845                                "id": "write",
6846                                "content": "Write /app/result.txt",
6847                                "status": "completed",
6848                                "evidence": "wrote answer"
6849                            }
6850                        ],
6851                        "requireCompletionEvidence": true
6852                    })
6853                    .to_string(),
6854                }))],
6855                _ => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
6856                    text: "final".to_string(),
6857                    phase: None,
6858                }))],
6859            };
6860            Ok(Box::pin(stream::iter(events.into_iter().chain(
6861                std::iter::once(Ok(InferenceEvent::Completed(CompletionMetadata {
6862                    stop_reason: Some("stop".to_string()),
6863                    provider_response_id: None,
6864                }))),
6865            ))))
6866        }
6867    }
6868
6869    struct VerificationGateEngine {
6870        calls: StdMutex<u32>,
6871    }
6872
6873    #[async_trait::async_trait]
6874    impl InferenceEngine for VerificationGateEngine {
6875        fn id(&self) -> String {
6876            roder_api::catalog::PROVIDER_MOCK.to_string()
6877        }
6878
6879        fn capabilities(&self) -> InferenceCapabilities {
6880            InferenceCapabilities::coding_agent_default()
6881        }
6882
6883        async fn list_models(
6884            &self,
6885            _ctx: InferenceProviderContext<'_>,
6886        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
6887            Ok(roder_api::catalog::models_for_provider(
6888                roder_api::catalog::PROVIDER_MOCK,
6889                true,
6890            ))
6891        }
6892
6893        async fn stream_turn(
6894            &self,
6895            _ctx: InferenceTurnContext<'_>,
6896            request: AgentInferenceRequest,
6897        ) -> anyhow::Result<InferenceEventStream> {
6898            let mut calls = self.calls.lock().unwrap();
6899            *calls += 1;
6900            let events = match *calls {
6901                1 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
6902                    id: "write-1".to_string(),
6903                    name: "write_file".to_string(),
6904                    arguments: serde_json::json!({
6905                        "path": "src/lib.rs",
6906                        "content": "pub fn answer() -> u8 { 42 }\n"
6907                    })
6908                    .to_string(),
6909                }))],
6910                2 => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
6911                    text: "done too early".to_string(),
6912                    phase: None,
6913                }))],
6914                3 if request.transcript.iter().any(|item| {
6915                    matches!(
6916                        item,
6917                        TranscriptItem::UserMessage(message)
6918                            if message.text.contains("Verification gate blocked final completion")
6919                    )
6920                }) =>
6921                {
6922                    vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
6923                        id: "verify-1".to_string(),
6924                        name: crate::verification_gate::VERIFICATION_TOOL_NAME.to_string(),
6925                        arguments: serde_json::json!({
6926                            "originalTask": "write code",
6927                            "changedFiles": ["src/lib.rs"],
6928                            "toolEvidence": ["write_file wrote src/lib.rs"],
6929                            "testsRun": ["cargo test -p roder-core verification_gate"],
6930                            "openGaps": [],
6931                            "status": "completed"
6932                        })
6933                        .to_string(),
6934                    }))]
6935                }
6936                _ => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
6937                    text: "verified final".to_string(),
6938                    phase: None,
6939                }))],
6940            };
6941            Ok(Box::pin(stream::iter(events.into_iter().chain(
6942                std::iter::once(Ok(InferenceEvent::Completed(CompletionMetadata {
6943                    stop_reason: Some("stop".to_string()),
6944                    provider_response_id: None,
6945                }))),
6946            ))))
6947        }
6948    }
6949
6950    struct SpeedPolicyEngine {
6951        calls: StdMutex<u32>,
6952        requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
6953    }
6954
6955    #[async_trait::async_trait]
6956    impl InferenceEngine for SpeedPolicyEngine {
6957        fn id(&self) -> String {
6958            roder_api::catalog::PROVIDER_MOCK.to_string()
6959        }
6960
6961        fn capabilities(&self) -> InferenceCapabilities {
6962            InferenceCapabilities::coding_agent_default()
6963        }
6964
6965        async fn list_models(
6966            &self,
6967            _ctx: InferenceProviderContext<'_>,
6968        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
6969            Ok(roder_api::catalog::models_for_provider(
6970                roder_api::catalog::PROVIDER_MOCK,
6971                true,
6972            ))
6973        }
6974
6975        async fn stream_turn(
6976            &self,
6977            _ctx: InferenceTurnContext<'_>,
6978            request: AgentInferenceRequest,
6979        ) -> anyhow::Result<InferenceEventStream> {
6980            self.requests.lock().unwrap().push(request.clone());
6981            let mut calls = self.calls.lock().unwrap();
6982            *calls += 1;
6983            let events = match *calls {
6984                1 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
6985                    id: "write-1".to_string(),
6986                    name: "write_file".to_string(),
6987                    arguments: serde_json::json!({
6988                        "path": "src/lib.rs",
6989                        "content": "pub fn answer() -> u8 { 42 }\n"
6990                    })
6991                    .to_string(),
6992                }))],
6993                3 if request.transcript.iter().any(|item| {
6994                    matches!(
6995                        item,
6996                        TranscriptItem::UserMessage(message)
6997                            if message.text.contains("Verification gate blocked final completion")
6998                    )
6999                }) =>
7000                {
7001                    vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7002                        id: "verify-1".to_string(),
7003                        name: crate::verification_gate::VERIFICATION_TOOL_NAME.to_string(),
7004                        arguments: serde_json::json!({
7005                            "originalTask": "write code",
7006                            "changedFiles": ["src/lib.rs"],
7007                            "toolEvidence": ["write_file wrote src/lib.rs"],
7008                            "testsRun": ["cargo test -p roder-core speed_policy"],
7009                            "openGaps": [],
7010                            "status": "completed"
7011                        })
7012                        .to_string(),
7013                    }))]
7014                }
7015                _ => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
7016                    text: "done".to_string(),
7017                    phase: None,
7018                }))],
7019            };
7020            Ok(Box::pin(stream::iter(events.into_iter().chain(
7021                std::iter::once(Ok(InferenceEvent::Completed(CompletionMetadata {
7022                    stop_reason: Some("stop".to_string()),
7023                    provider_response_id: None,
7024                }))),
7025            ))))
7026        }
7027    }
7028
7029    struct SwitchCaptureEngine {
7030        requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
7031    }
7032
7033    #[async_trait::async_trait]
7034    impl InferenceEngine for SwitchCaptureEngine {
7035        fn id(&self) -> String {
7036            roder_api::catalog::PROVIDER_MOCK.to_string()
7037        }
7038
7039        fn capabilities(&self) -> InferenceCapabilities {
7040            InferenceCapabilities::coding_agent_default()
7041        }
7042
7043        async fn list_models(
7044            &self,
7045            _ctx: InferenceProviderContext<'_>,
7046        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7047            Ok(roder_api::catalog::models_for_provider(
7048                roder_api::catalog::PROVIDER_MOCK,
7049                true,
7050            ))
7051        }
7052
7053        async fn stream_turn(
7054            &self,
7055            _ctx: InferenceTurnContext<'_>,
7056            request: AgentInferenceRequest,
7057        ) -> anyhow::Result<InferenceEventStream> {
7058            self.requests.lock().unwrap().push(request);
7059            Ok(Box::pin(stream::iter(vec![
7060                Ok(InferenceEvent::MessageDelta(MessageDelta {
7061                    text: "done".to_string(),
7062                    phase: None,
7063                })),
7064                Ok(InferenceEvent::Completed(CompletionMetadata {
7065                    stop_reason: Some("stop".to_string()),
7066                    provider_response_id: None,
7067                })),
7068            ])))
7069        }
7070    }
7071
7072    struct DeadlineEngine;
7073
7074    #[async_trait::async_trait]
7075    impl InferenceEngine for DeadlineEngine {
7076        fn id(&self) -> String {
7077            roder_api::catalog::PROVIDER_MOCK.to_string()
7078        }
7079
7080        fn capabilities(&self) -> InferenceCapabilities {
7081            InferenceCapabilities::coding_agent_default()
7082        }
7083
7084        async fn list_models(
7085            &self,
7086            _ctx: InferenceProviderContext<'_>,
7087        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7088            Ok(Vec::new())
7089        }
7090
7091        async fn stream_turn(
7092            &self,
7093            _ctx: InferenceTurnContext<'_>,
7094            _request: AgentInferenceRequest,
7095        ) -> anyhow::Result<InferenceEventStream> {
7096            Ok(Box::pin(stream::once(async {
7097                tokio::time::sleep(std::time::Duration::from_secs(60)).await;
7098                Ok(InferenceEvent::MessageDelta(MessageDelta {
7099                    text: "too late".to_string(),
7100                    phase: None,
7101                }))
7102            })))
7103        }
7104    }
7105
7106    struct WriteFileContributor;
7107
7108    impl ToolContributor for WriteFileContributor {
7109        fn id(&self) -> String {
7110            "test-write".to_string()
7111        }
7112
7113        fn contribute(&self, registry: &mut ToolRegistry) -> anyhow::Result<()> {
7114            registry.register(Arc::new(WriteFileTool))
7115        }
7116    }
7117
7118    struct WriteFileTool;
7119
7120    #[async_trait::async_trait]
7121    impl ToolExecutor for WriteFileTool {
7122        fn spec(&self) -> ToolSpec {
7123            ToolSpec {
7124                name: "write_file".to_string(),
7125                description: "Write a test file.".to_string(),
7126                parameters: serde_json::json!({
7127                    "type": "object",
7128                    "properties": {
7129                        "path": { "type": "string" },
7130                        "content": { "type": "string" }
7131                    },
7132                    "required": ["path", "content"],
7133                    "additionalProperties": false
7134                }),
7135            }
7136        }
7137
7138        async fn execute(
7139            &self,
7140            _ctx: ToolExecutionContext,
7141            call: ToolCall,
7142        ) -> anyhow::Result<ToolResult> {
7143            let path = call
7144                .arguments
7145                .get("path")
7146                .and_then(serde_json::Value::as_str)
7147                .unwrap_or("src/lib.rs");
7148            Ok(ToolResult {
7149                id: call.id,
7150                name: call.name,
7151                text: format!("wrote {path}"),
7152                data: serde_json::json!({ "path": path }),
7153                is_error: false,
7154            })
7155        }
7156    }
7157
7158    struct ProfileToolContributor;
7159
7160    impl ToolContributor for ProfileToolContributor {
7161        fn id(&self) -> String {
7162            "profile-tools".to_string()
7163        }
7164
7165        fn contribute(&self, registry: &mut ToolRegistry) -> anyhow::Result<()> {
7166            for name in ["apply_patch", "edit", "multi_edit", "write_file"] {
7167                registry.register(Arc::new(ProfileTool {
7168                    name: name.to_string(),
7169                }))?;
7170            }
7171            Ok(())
7172        }
7173    }
7174
7175    struct ProfileTool {
7176        name: String,
7177    }
7178
7179    #[async_trait::async_trait]
7180    impl ToolExecutor for ProfileTool {
7181        fn spec(&self) -> ToolSpec {
7182            ToolSpec {
7183                name: self.name.clone(),
7184                description: format!("{} test tool", self.name),
7185                parameters: serde_json::json!({
7186                    "type": "object",
7187                    "properties": {
7188                        "path": { "type": "string" },
7189                        "content": { "type": "string" }
7190                    },
7191                    "required": ["path", "content"],
7192                    "additionalProperties": false
7193                }),
7194            }
7195        }
7196
7197        async fn execute(
7198            &self,
7199            _ctx: ToolExecutionContext,
7200            call: ToolCall,
7201        ) -> anyhow::Result<ToolResult> {
7202            Ok(ToolResult {
7203                id: call.id,
7204                name: call.name,
7205                text: "ok".to_string(),
7206                data: serde_json::json!({}),
7207                is_error: false,
7208            })
7209        }
7210    }
7211
7212    fn test_model_profile(model: &str) -> ModelHarnessProfile {
7213        ModelHarnessProfile {
7214            model: model.to_string(),
7215            provider: roder_api::catalog::PROVIDER_OPENAI.to_string(),
7216            provider_family: ProviderFamily::OpenAi,
7217            edit_tool: Some(EDIT_TOOL_EDIT.to_string()),
7218            schema_policy: ModelSchemaPolicy::StandardRequiredFirst,
7219            instruction_overlay: ModelInstructionOverlay::IntuitiveContext,
7220            reasoning: ModelProfileReasoning {
7221                orientation: Some(REASONING_LOW.to_string()),
7222                execution: Some(REASONING_LOW.to_string()),
7223                verification: Some(REASONING_LOW.to_string()),
7224                recovery: Some(REASONING_LOW.to_string()),
7225            },
7226            parallel_tool_calls: Some(false),
7227            auto_compact_token_limit: Some(123_000),
7228        }
7229    }
7230
7231    async fn captured_profile_request(cfg: RuntimeConfig) -> AgentInferenceRequest {
7232        let captured = Arc::new(StdMutex::new(None));
7233        let mut builder = ExtensionRegistryBuilder::new();
7234        builder.inference_engine(Arc::new(CapturingEngine {
7235            request: captured.clone(),
7236        }));
7237        builder.tool_contributor(Arc::new(ProfileToolContributor));
7238        let runtime = Arc::new(Runtime::new(builder.build().unwrap(), cfg).unwrap());
7239        let mut rx = runtime.subscribe_events();
7240        let turn_id = runtime
7241            .start_turn(StartTurnRequest {
7242                thread_id: "thread-model-profile".to_string(),
7243                message: "use profile knobs".to_string(),
7244                images: Vec::new(),
7245                provider_override: None,
7246                model_override: None,
7247                reasoning_override: None,
7248                workspace: test_workspace(),
7249                instructions: InstructionBundle {
7250                    system: None,
7251                    developer: Some("base developer".to_string()),
7252                    developer_context: None,
7253                },
7254                developer_context: None,
7255                task_ledger_required: false,
7256            })
7257            .await
7258            .unwrap();
7259
7260        tokio::time::timeout(std::time::Duration::from_secs(5), async {
7261            loop {
7262                let envelope = rx.recv().await.unwrap();
7263                if envelope.turn_id.as_deref() != Some(&turn_id) {
7264                    continue;
7265                }
7266                match envelope.event {
7267                    RoderEvent::TurnCompleted(_) => break,
7268                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7269                    _ => {}
7270                }
7271            }
7272        })
7273        .await
7274        .unwrap();
7275
7276        captured.lock().unwrap().clone().unwrap()
7277    }
7278
7279    struct ToolThenStopEngine {
7280        calls: StdMutex<u32>,
7281    }
7282
7283    #[async_trait::async_trait]
7284    impl InferenceEngine for ToolThenStopEngine {
7285        fn id(&self) -> String {
7286            roder_api::catalog::PROVIDER_MOCK.to_string()
7287        }
7288
7289        fn capabilities(&self) -> InferenceCapabilities {
7290            InferenceCapabilities::coding_agent_default()
7291        }
7292
7293        async fn list_models(
7294            &self,
7295            _ctx: InferenceProviderContext<'_>,
7296        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7297            Ok(roder_api::catalog::models_for_provider(
7298                roder_api::catalog::PROVIDER_MOCK,
7299                true,
7300            ))
7301        }
7302
7303        async fn stream_turn(
7304            &self,
7305            _ctx: InferenceTurnContext<'_>,
7306            _request: AgentInferenceRequest,
7307        ) -> anyhow::Result<InferenceEventStream> {
7308            let mut calls = self.calls.lock().unwrap();
7309            *calls += 1;
7310            let events = match *calls {
7311                1 => vec![
7312                    Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7313                        id: "write-1".to_string(),
7314                        name: "write_file".to_string(),
7315                        arguments: serde_json::json!({
7316                            "path": "src/lib.rs",
7317                            "content": "pub fn answer() -> u8 { 42 }\n"
7318                        })
7319                        .to_string(),
7320                    })),
7321                    Ok(InferenceEvent::Completed(CompletionMetadata {
7322                        stop_reason: Some("tool_use".to_string()),
7323                        provider_response_id: None,
7324                    })),
7325                ],
7326                _ => vec![
7327                    Ok(InferenceEvent::MessageDelta(MessageDelta {
7328                        text: "final".to_string(),
7329                        phase: None,
7330                    })),
7331                    Ok(InferenceEvent::Completed(CompletionMetadata {
7332                        stop_reason: Some("end_turn".to_string()),
7333                        provider_response_id: None,
7334                    })),
7335                ],
7336            };
7337            Ok(Box::pin(stream::iter(events)))
7338        }
7339    }
7340
7341    #[tokio::test]
7342    async fn turn_completed_reports_terminal_step_finish_reason() {
7343        let mut builder = ExtensionRegistryBuilder::new();
7344        builder.inference_engine(Arc::new(ToolThenStopEngine {
7345            calls: StdMutex::new(0),
7346        }));
7347        builder.tool_contributor(Arc::new(WriteFileContributor));
7348        let runtime = Arc::new(
7349            Runtime::new(
7350                builder.build().unwrap(),
7351                RuntimeConfig {
7352                    policy_mode: PolicyMode::Bypass,
7353                    agent_swarm_mode: false,
7354                    ..RuntimeConfig::default()
7355                },
7356            )
7357            .unwrap(),
7358        );
7359        let mut rx = runtime.subscribe_events();
7360        let turn_id = runtime
7361            .start_turn(StartTurnRequest {
7362                thread_id: "thread-finish-reason".to_string(),
7363                message: "write then finish".to_string(),
7364                images: Vec::new(),
7365                provider_override: None,
7366                model_override: None,
7367                reasoning_override: None,
7368                workspace: test_workspace(),
7369                instructions: InstructionBundle {
7370                    system: None,
7371                    developer: None,
7372                    developer_context: None,
7373                },
7374                developer_context: None,
7375                task_ledger_required: false,
7376            })
7377            .await
7378            .unwrap();
7379
7380        let completed = tokio::time::timeout(std::time::Duration::from_secs(5), async {
7381            loop {
7382                let envelope = rx.recv().await.unwrap();
7383                if envelope.turn_id.as_deref() != Some(&turn_id) {
7384                    continue;
7385                }
7386                match envelope.event {
7387                    RoderEvent::TurnCompleted(event) => break event,
7388                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7389                    _ => {}
7390                }
7391            }
7392        })
7393        .await
7394        .unwrap();
7395
7396        // The mid-turn tool_use stop reason must not leak; the terminal
7397        // end_turn step decides the turn's finish reason.
7398        assert_eq!(completed.finish_reason.as_deref(), Some("stop"));
7399    }
7400
7401    #[tokio::test]
7402    async fn inference_router_selection_changes_request_model_and_records_event() {
7403        let default_requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7404        let routed_requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7405        let contexts = Arc::new(StdMutex::new(Vec::<InferenceRoutingContext>::new()));
7406        let selected = ModelSelection {
7407            provider: "routed-provider".to_string(),
7408            model: "routed-model".to_string(),
7409        };
7410        let default = ModelSelection {
7411            provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7412            model: "mock".to_string(),
7413        };
7414        let decision = InferenceRoutingDecision {
7415            reasoning: Some(ReasoningConfig {
7416                enabled: true,
7417                level: Some(REASONING_LOW.to_string()),
7418            }),
7419            confidence: Some(0.91),
7420            baseline: Some(default.clone()),
7421            matched_signals: vec![roder_api::inference_routing::InferenceRoutingSignal::new(
7422                "intent", "routine",
7423            )],
7424            ..InferenceRoutingDecision::selected("test-router", selected.clone(), "routine request")
7425        };
7426
7427        let mut builder = ExtensionRegistryBuilder::new();
7428        builder.inference_engine(Arc::new(RoutingCaptureEngine {
7429            id: roder_api::catalog::PROVIDER_MOCK,
7430            models: vec![routing_test_model("mock", &[REASONING_LOW])],
7431            requests: default_requests.clone(),
7432        }));
7433        builder.inference_engine(Arc::new(RoutingCaptureEngine {
7434            id: "routed-provider",
7435            models: vec![routing_test_model(
7436                "routed-model",
7437                &[REASONING_LOW, REASONING_MEDIUM],
7438            )],
7439            requests: routed_requests.clone(),
7440        }));
7441        builder.inference_router(Arc::new(StaticRouter {
7442            id: "test-router",
7443            decision,
7444            contexts: contexts.clone(),
7445        }));
7446        let thread_root =
7447            std::env::temp_dir().join(format!("roder-routing-auto-{}", uuid::Uuid::new_v4()));
7448        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
7449            base_path: thread_root.clone(),
7450        }));
7451        let runtime = Arc::new(
7452            Runtime::new(
7453                builder.build().unwrap(),
7454                RuntimeConfig {
7455                    default_provider: default.provider.clone(),
7456                    default_model: default.model.clone(),
7457                    ..RuntimeConfig::default()
7458                },
7459            )
7460            .unwrap(),
7461        );
7462        let thread_id = runtime
7463            .create_thread_with(CreateThreadRequest {
7464                title: Some("Routing auto".to_string()),
7465                workspace: test_workspace(),
7466                workspace_id: None,
7467                root_id: None,
7468                provider: Some(default.provider.clone()),
7469                model: Some(default.model.clone()),
7470                tool_allowlist: Vec::new(),
7471                developer_instructions: None,
7472                external_tools: Vec::new(),
7473                selection_mode: Some(ModelSelectionMode::auto(
7474                    "test-router:coding",
7475                    "test-router",
7476                    "Auto: Coding",
7477                    default.clone(),
7478                    Some("coding".to_string()),
7479                    None,
7480                )),
7481                runner: None,
7482            })
7483            .await
7484            .unwrap()
7485            .thread_id;
7486        let mut rx = runtime.subscribe_events();
7487        let turn_id = runtime
7488            .start_turn(StartTurnRequest {
7489                thread_id: thread_id.clone(),
7490                message: "small cleanup".to_string(),
7491                images: Vec::new(),
7492                provider_override: None,
7493                model_override: None,
7494                reasoning_override: None,
7495                workspace: test_workspace(),
7496                instructions: InstructionBundle::default(),
7497                developer_context: None,
7498                task_ledger_required: false,
7499            })
7500            .await
7501            .unwrap();
7502
7503        let mut routing_event = None;
7504        let mut inference_started = None;
7505        tokio::time::timeout(std::time::Duration::from_secs(5), async {
7506            loop {
7507                let envelope = rx.recv().await.unwrap();
7508                if envelope.turn_id.as_deref() != Some(&turn_id) {
7509                    continue;
7510                }
7511                match envelope.event {
7512                    RoderEvent::InferenceRoutingDecision(event) => {
7513                        routing_event = Some(event);
7514                    }
7515                    RoderEvent::InferenceStarted(event) => {
7516                        inference_started = Some(event);
7517                    }
7518                    RoderEvent::TurnCompleted(_) => break,
7519                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7520                    _ => {}
7521                }
7522            }
7523        })
7524        .await
7525        .unwrap();
7526
7527        assert!(default_requests.lock().unwrap().is_empty());
7528        let routed_requests = routed_requests.lock().unwrap();
7529        assert_eq!(routed_requests.len(), 1);
7530        assert_eq!(routed_requests[0].model, selected);
7531        assert_eq!(
7532            routed_requests[0].reasoning.level.as_deref(),
7533            Some(REASONING_LOW)
7534        );
7535        assert_eq!(
7536            routed_requests[0].metadata["inferenceRouting"]["outcome"],
7537            "selected"
7538        );
7539
7540        let routing_event = routing_event.expect("routing decision event");
7541        assert_eq!(routing_event.default_selection, default);
7542        assert_eq!(routing_event.selected_selection, selected);
7543        assert_eq!(
7544            routing_event.decision.outcome,
7545            InferenceRoutingOutcome::Selected
7546        );
7547        assert_eq!(
7548            inference_started.expect("inference started event").model,
7549            selected
7550        );
7551
7552        let contexts = contexts.lock().unwrap();
7553        assert_eq!(contexts.len(), 1);
7554        assert_eq!(contexts[0].default_selection, default);
7555        assert_eq!(contexts[0].candidates.len(), 2);
7556        assert!(
7557            contexts[0]
7558                .signals
7559                .iter()
7560                .any(|signal| signal.key == "profile" && signal.value == "coding")
7561        );
7562        let _ = std::fs::remove_dir_all(thread_root);
7563    }
7564
7565    #[tokio::test]
7566    async fn inference_router_is_bypassed_for_explicit_selection() {
7567        let requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7568        let contexts = Arc::new(StdMutex::new(Vec::<InferenceRoutingContext>::new()));
7569        let selected = ModelSelection {
7570            provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7571            model: "mock".to_string(),
7572        };
7573
7574        let mut builder = ExtensionRegistryBuilder::new();
7575        builder.inference_engine(Arc::new(RoutingCaptureEngine {
7576            id: roder_api::catalog::PROVIDER_MOCK,
7577            models: vec![routing_test_model("mock", &[REASONING_LOW])],
7578            requests: requests.clone(),
7579        }));
7580        builder.inference_router(Arc::new(StaticRouter {
7581            id: "test-router",
7582            decision: InferenceRoutingDecision::selected(
7583                "test-router",
7584                ModelSelection {
7585                    provider: "missing".to_string(),
7586                    model: "missing".to_string(),
7587                },
7588                "would route if called",
7589            ),
7590            contexts: contexts.clone(),
7591        }));
7592        let thread_root =
7593            std::env::temp_dir().join(format!("roder-routing-explicit-{}", uuid::Uuid::new_v4()));
7594        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
7595            base_path: thread_root.clone(),
7596        }));
7597        let runtime = Arc::new(
7598            Runtime::new(
7599                builder.build().unwrap(),
7600                RuntimeConfig {
7601                    default_provider: selected.provider.clone(),
7602                    default_model: selected.model.clone(),
7603                    inference_router: RuntimeInferenceRouterConfig {
7604                        enabled: true,
7605                        router_id: Some("test-router".to_string()),
7606                    },
7607                    ..RuntimeConfig::default()
7608                },
7609            )
7610            .unwrap(),
7611        );
7612        let thread_id = runtime
7613            .create_thread_with(CreateThreadRequest {
7614                title: Some("Routing explicit".to_string()),
7615                workspace: test_workspace(),
7616                workspace_id: None,
7617                root_id: None,
7618                provider: Some(selected.provider.clone()),
7619                model: Some(selected.model.clone()),
7620                tool_allowlist: Vec::new(),
7621                developer_instructions: None,
7622                external_tools: Vec::new(),
7623                selection_mode: Some(ModelSelectionMode::auto(
7624                    "test-router:default",
7625                    "test-router",
7626                    "Auto",
7627                    selected.clone(),
7628                    None,
7629                    None,
7630                )),
7631                runner: None,
7632            })
7633            .await
7634            .unwrap()
7635            .thread_id;
7636        let mut rx = runtime.subscribe_events();
7637        let turn_id = runtime
7638            .start_turn(StartTurnRequest {
7639                thread_id,
7640                message: "use explicit selection".to_string(),
7641                images: Vec::new(),
7642                provider_override: Some(selected.provider.clone()),
7643                model_override: Some(selected.model.clone()),
7644                reasoning_override: None,
7645                workspace: test_workspace(),
7646                instructions: InstructionBundle::default(),
7647                developer_context: None,
7648                task_ledger_required: false,
7649            })
7650            .await
7651            .unwrap();
7652
7653        let mut saw_routing_event = false;
7654        tokio::time::timeout(std::time::Duration::from_secs(5), async {
7655            loop {
7656                let envelope = rx.recv().await.unwrap();
7657                if envelope.turn_id.as_deref() != Some(&turn_id) {
7658                    continue;
7659                }
7660                match envelope.event {
7661                    RoderEvent::InferenceRoutingDecision(_) => {
7662                        saw_routing_event = true;
7663                    }
7664                    RoderEvent::TurnCompleted(_) => break,
7665                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7666                    _ => {}
7667                }
7668            }
7669        })
7670        .await
7671        .unwrap();
7672
7673        assert!(!saw_routing_event);
7674        assert!(contexts.lock().unwrap().is_empty());
7675        let requests = requests.lock().unwrap();
7676        assert_eq!(requests.len(), 1);
7677        assert_eq!(requests[0].model, selected);
7678        let _ = std::fs::remove_dir_all(thread_root);
7679    }
7680
7681    #[tokio::test]
7682    async fn inference_router_is_bypassed_for_manual_selection_mode() {
7683        let requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7684        let contexts = Arc::new(StdMutex::new(Vec::<InferenceRoutingContext>::new()));
7685        let selected = ModelSelection {
7686            provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7687            model: "mock".to_string(),
7688        };
7689
7690        let mut builder = ExtensionRegistryBuilder::new();
7691        builder.inference_engine(Arc::new(RoutingCaptureEngine {
7692            id: roder_api::catalog::PROVIDER_MOCK,
7693            models: vec![routing_test_model("mock", &[REASONING_LOW])],
7694            requests: requests.clone(),
7695        }));
7696        builder.inference_router(Arc::new(StaticRouter {
7697            id: "test-router",
7698            decision: InferenceRoutingDecision::selected(
7699                "test-router",
7700                ModelSelection {
7701                    provider: "missing".to_string(),
7702                    model: "missing".to_string(),
7703                },
7704                "would route if called",
7705            ),
7706            contexts: contexts.clone(),
7707        }));
7708        let thread_root =
7709            std::env::temp_dir().join(format!("roder-routing-manual-{}", uuid::Uuid::new_v4()));
7710        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
7711            base_path: thread_root.clone(),
7712        }));
7713        let runtime = Arc::new(
7714            Runtime::new(
7715                builder.build().unwrap(),
7716                RuntimeConfig {
7717                    default_provider: selected.provider.clone(),
7718                    default_model: selected.model.clone(),
7719                    inference_router: RuntimeInferenceRouterConfig {
7720                        enabled: true,
7721                        router_id: Some("test-router".to_string()),
7722                    },
7723                    ..RuntimeConfig::default()
7724                },
7725            )
7726            .unwrap(),
7727        );
7728        let thread_id = runtime
7729            .create_thread_with(CreateThreadRequest {
7730                title: Some("Routing manual".to_string()),
7731                workspace: test_workspace(),
7732                workspace_id: None,
7733                root_id: None,
7734                provider: Some(selected.provider.clone()),
7735                model: Some(selected.model.clone()),
7736                tool_allowlist: Vec::new(),
7737                developer_instructions: None,
7738                external_tools: Vec::new(),
7739                selection_mode: Some(ModelSelectionMode::manual(
7740                    selected.provider.clone(),
7741                    selected.model.clone(),
7742                    None,
7743                )),
7744                runner: None,
7745            })
7746            .await
7747            .unwrap()
7748            .thread_id;
7749        let mut rx = runtime.subscribe_events();
7750        let turn_id = runtime
7751            .start_turn(StartTurnRequest {
7752                thread_id,
7753                message: "use selected manual model".to_string(),
7754                images: Vec::new(),
7755                provider_override: None,
7756                model_override: None,
7757                reasoning_override: None,
7758                workspace: test_workspace(),
7759                instructions: InstructionBundle::default(),
7760                developer_context: None,
7761                task_ledger_required: false,
7762            })
7763            .await
7764            .unwrap();
7765
7766        let mut saw_routing_event = false;
7767        tokio::time::timeout(std::time::Duration::from_secs(5), async {
7768            loop {
7769                let envelope = rx.recv().await.unwrap();
7770                if envelope.turn_id.as_deref() != Some(&turn_id) {
7771                    continue;
7772                }
7773                match envelope.event {
7774                    RoderEvent::InferenceRoutingDecision(_) => {
7775                        saw_routing_event = true;
7776                    }
7777                    RoderEvent::TurnCompleted(_) => break,
7778                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7779                    _ => {}
7780                }
7781            }
7782        })
7783        .await
7784        .unwrap();
7785
7786        assert!(!saw_routing_event);
7787        assert!(contexts.lock().unwrap().is_empty());
7788        let requests = requests.lock().unwrap();
7789        assert_eq!(requests.len(), 1);
7790        assert_eq!(requests[0].model, selected);
7791        let _ = std::fs::remove_dir_all(thread_root);
7792    }
7793
7794    #[test]
7795    fn enabled_inference_router_requires_registered_router() {
7796        let mut builder = ExtensionRegistryBuilder::new();
7797        builder.inference_engine(Arc::new(FakeInferenceEngine));
7798
7799        let err = match Runtime::new(
7800            builder.build().unwrap(),
7801            RuntimeConfig {
7802                inference_router: RuntimeInferenceRouterConfig {
7803                    enabled: true,
7804                    router_id: Some("missing-router".to_string()),
7805                },
7806                ..RuntimeConfig::default()
7807            },
7808        ) {
7809            Ok(_) => panic!("runtime should reject unknown inference router"),
7810            Err(err) => err,
7811        };
7812
7813        assert!(
7814            err.to_string()
7815                .contains("inference router \"missing-router\" is not registered")
7816        );
7817    }
7818
7819    #[tokio::test]
7820    async fn model_profile_routes_request_knobs_to_next_inference() {
7821        let request = captured_profile_request(RuntimeConfig {
7822            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7823            default_model: "gpt-5.5".to_string(),
7824            model_profiles: std::collections::HashMap::from([(
7825                "gpt-5.5".to_string(),
7826                test_model_profile("gpt-5.5"),
7827            )]),
7828            ..RuntimeConfig::default()
7829        })
7830        .await;
7831
7832        let tool_names = request
7833            .tools
7834            .iter()
7835            .map(|tool| tool.name.as_str())
7836            .collect::<Vec<_>>();
7837        assert!(tool_names.contains(&"apply_patch"));
7838        assert!(tool_names.contains(&"edit"));
7839        assert!(tool_names.contains(&"multi_edit"));
7840        assert!(tool_names.contains(&"write_file"));
7841        assert_eq!(request.reasoning.level.as_deref(), Some(REASONING_LOW));
7842        assert_eq!(request.runtime.parallel_tool_calls, Some(false));
7843        assert_eq!(request.runtime.auto_compact_token_limit, Some(123_000));
7844        assert!(
7845            request
7846                .instructions
7847                .developer
7848                .as_deref()
7849                .unwrap_or_default()
7850                .contains("Use the provided context as the current working set")
7851        );
7852        assert_eq!(
7853            request
7854                .metadata
7855                .pointer("/modelProfile/schemaPolicy")
7856                .and_then(serde_json::Value::as_str),
7857            Some("standard_required_first")
7858        );
7859    }
7860
7861    #[tokio::test]
7862    async fn ultra_reasoning_reaches_transport_state_and_enables_proactive_delegation() {
7863        let request = captured_profile_request(RuntimeConfig {
7864            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7865            default_model: "gpt-5.6-sol".to_string(),
7866            reasoning: Some(REASONING_ULTRA.to_string()),
7867            ..RuntimeConfig::default()
7868        })
7869        .await;
7870
7871        assert_eq!(request.reasoning.level.as_deref(), Some(REASONING_ULTRA));
7872        let developer = request
7873            .instructions
7874            .developer
7875            .as_deref()
7876            .expect("ultra developer instructions");
7877        assert!(developer.contains("Proactive multi-agent delegation is active"));
7878    }
7879
7880    #[tokio::test]
7881    async fn lower_sol_effort_requires_explicit_multi_agent_request() {
7882        let request = captured_profile_request(RuntimeConfig {
7883            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7884            default_model: "gpt-5.6-sol".to_string(),
7885            reasoning: Some(roder_api::catalog::REASONING_MEDIUM.to_string()),
7886            ..RuntimeConfig::default()
7887        })
7888        .await;
7889
7890        let developer = request
7891            .instructions
7892            .developer
7893            .as_deref()
7894            .expect("sol developer instructions");
7895        assert!(developer.contains("Do not spawn sub-agents unless"));
7896        assert!(!developer.contains("Proactive multi-agent delegation is active"));
7897    }
7898
7899    #[tokio::test]
7900    async fn luna_does_not_receive_codex_v2_multi_agent_policy() {
7901        let request = captured_profile_request(RuntimeConfig {
7902            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7903            default_model: "gpt-5.6-luna".to_string(),
7904            reasoning: Some(roder_api::catalog::REASONING_MAX.to_string()),
7905            ..RuntimeConfig::default()
7906        })
7907        .await;
7908
7909        let developer = request
7910            .instructions
7911            .developer
7912            .as_deref()
7913            .unwrap_or_default();
7914        assert!(!developer.contains("Do not spawn sub-agents unless"));
7915        assert!(!developer.contains("Proactive multi-agent delegation is active"));
7916    }
7917
7918    #[tokio::test]
7919    async fn turn_developer_context_reaches_inference_and_does_not_persist() {
7920        let captured = Arc::new(StdMutex::new(None));
7921        let mut builder = ExtensionRegistryBuilder::new();
7922        builder.inference_engine(Arc::new(CapturingEngine {
7923            request: captured.clone(),
7924        }));
7925        let runtime = Arc::new(
7926            Runtime::new(
7927                builder.build().unwrap(),
7928                RuntimeConfig {
7929                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7930                    default_model: "gpt-5.5".to_string(),
7931                    ..RuntimeConfig::default()
7932                },
7933            )
7934            .unwrap(),
7935        );
7936
7937        async fn run_turn(runtime: &Arc<Runtime>, developer_context: Option<String>) {
7938            let mut rx = runtime.subscribe_events();
7939            let turn_id = runtime
7940                .start_turn(StartTurnRequest {
7941                    thread_id: "thread-turn-context".to_string(),
7942                    message: "hello".to_string(),
7943                    images: Vec::new(),
7944                    provider_override: None,
7945                    model_override: None,
7946                    reasoning_override: None,
7947                    workspace: test_workspace(),
7948                    instructions: InstructionBundle::default(),
7949                    developer_context,
7950                    task_ledger_required: false,
7951                })
7952                .await
7953                .unwrap();
7954            tokio::time::timeout(std::time::Duration::from_secs(5), async {
7955                loop {
7956                    let envelope = rx.recv().await.unwrap();
7957                    if envelope.turn_id.as_deref() != Some(&turn_id) {
7958                        continue;
7959                    }
7960                    match envelope.event {
7961                        RoderEvent::TurnCompleted(_) => break,
7962                        RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7963                        _ => {}
7964                    }
7965                }
7966            })
7967            .await
7968            .unwrap();
7969        }
7970
7971        run_turn(
7972            &runtime,
7973            Some("Connected accounts: example-service.".to_string()),
7974        )
7975        .await;
7976        let request = captured.lock().unwrap().clone().unwrap();
7977        assert_eq!(
7978            request.instructions.developer_context.as_deref(),
7979            Some("Connected accounts: example-service.")
7980        );
7981
7982        // The context is per-turn only: the next turn on the same thread
7983        // without a developerContext must not see the previous one.
7984        run_turn(&runtime, None).await;
7985        let request = captured.lock().unwrap().clone().unwrap();
7986        assert_eq!(request.instructions.developer_context, None);
7987    }
7988
7989    #[tokio::test]
7990    async fn tool_search_overrides_route_to_next_inference_request() {
7991        let request = captured_profile_request(RuntimeConfig {
7992            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7993            default_model: "gpt-5.4".to_string(),
7994            tool_search: ToolSearchConfig {
7995                mode: roder_api::inference::ToolSearchMode::Auto,
7996                max_catalog_items: Some(100),
7997                ..ToolSearchConfig::default()
7998            },
7999            provider_tool_search: std::collections::HashMap::from([(
8000                roder_api::catalog::PROVIDER_MOCK.to_string(),
8001                roder_api::inference::ToolSearchConfigOverlay {
8002                    include_skills: Some(false),
8003                    provider_variant: Some(roder_api::inference::ToolSearchProviderVariant::Regex),
8004                    ..Default::default()
8005                },
8006            )]),
8007            model_tool_search: std::collections::HashMap::from([(
8008                "gpt-5.4".to_string(),
8009                roder_api::inference::ToolSearchConfigOverlay {
8010                    mode: Some(roder_api::inference::ToolSearchMode::ProviderNative),
8011                    max_catalog_items: Some(25),
8012                    provider_variant: Some(roder_api::inference::ToolSearchProviderVariant::Bm25),
8013                    ..Default::default()
8014                },
8015            )]),
8016            ..RuntimeConfig::default()
8017        })
8018        .await;
8019
8020        assert_eq!(
8021            request.runtime.tool_search.mode,
8022            roder_api::inference::ToolSearchMode::ProviderNative
8023        );
8024        assert_eq!(request.runtime.tool_search.max_catalog_items, Some(25));
8025        assert_eq!(
8026            request.runtime.tool_search.provider_variant,
8027            roder_api::inference::ToolSearchProviderVariant::Bm25
8028        );
8029    }
8030
8031    #[tokio::test]
8032    async fn context_entrypoint_hints_use_turn_workspace() {
8033        let process_workspace = runtime_test_workspace("entrypoint-process");
8034        let thread_workspace = runtime_test_workspace("entrypoint-thread");
8035        std::fs::create_dir_all(process_workspace.join("src")).unwrap();
8036        std::fs::create_dir_all(thread_workspace.join("src")).unwrap();
8037        std::fs::write(
8038            process_workspace.join("src/sidebar-thread-groups.ts"),
8039            "export const desktopLeak = true;\n",
8040        )
8041        .unwrap();
8042        std::fs::write(
8043            thread_workspace.join("src/voice-plan-feedback.ts"),
8044            "export const voicePlanFeedback = true;\n",
8045        )
8046        .unwrap();
8047
8048        let captured = Arc::new(StdMutex::new(None));
8049        let mut builder = ExtensionRegistryBuilder::new();
8050        builder.inference_engine(Arc::new(CapturingEngine {
8051            request: captured.clone(),
8052        }));
8053        builder.context_planner(Arc::new(roder_context::EntrypointContextPlanner::new(
8054            process_workspace.clone(),
8055        )));
8056        let runtime = Arc::new(
8057            Runtime::new(
8058                builder.build().unwrap(),
8059                RuntimeConfig {
8060                    workspace: Some(process_workspace.display().to_string()),
8061                    ..RuntimeConfig::default()
8062                },
8063            )
8064            .unwrap(),
8065        );
8066        let mut rx = runtime.subscribe_events();
8067
8068        let turn_id = runtime
8069            .start_turn(StartTurnRequest {
8070                thread_id: "thread-workspace-entrypoint".to_string(),
8071                message: "investigate voice plan feedback".to_string(),
8072                images: Vec::new(),
8073                provider_override: None,
8074                model_override: None,
8075                reasoning_override: None,
8076                workspace: thread_workspace.display().to_string(),
8077                instructions: crate::instructions::default_instructions(),
8078                developer_context: None,
8079                task_ledger_required: false,
8080            })
8081            .await
8082            .unwrap();
8083
8084        tokio::time::timeout(std::time::Duration::from_secs(5), async {
8085            loop {
8086                let envelope = rx.recv().await.unwrap();
8087                if envelope.turn_id.as_deref() != Some(&turn_id) {
8088                    continue;
8089                }
8090                match envelope.event {
8091                    RoderEvent::TurnCompleted(_) => break,
8092                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8093                    _ => {}
8094                }
8095            }
8096        })
8097        .await
8098        .unwrap();
8099
8100        let request = captured.lock().unwrap().clone().expect("captured request");
8101        let transcript_text = request
8102            .transcript
8103            .iter()
8104            .map(|item| match item {
8105                TranscriptItem::UserMessage(message) => message.text.as_str(),
8106                _ => "",
8107            })
8108            .collect::<Vec<_>>()
8109            .join("\n");
8110        assert!(transcript_text.contains("src/voice-plan-feedback.ts"));
8111        assert!(!transcript_text.contains("src/sidebar-thread-groups.ts"));
8112
8113        let _ = std::fs::remove_dir_all(process_workspace);
8114        let _ = std::fs::remove_dir_all(thread_workspace);
8115    }
8116
8117    fn runtime_test_workspace(name: &str) -> std::path::PathBuf {
8118        let path =
8119            std::env::temp_dir().join(format!("roder-runtime-{name}-{}", uuid::Uuid::new_v4()));
8120        let _ = std::fs::remove_dir_all(&path);
8121        std::fs::create_dir_all(&path).unwrap();
8122        path
8123    }
8124
8125    #[tokio::test]
8126    async fn model_profile_user_model_knobs_override_profile_defaults() {
8127        let request = captured_profile_request(RuntimeConfig {
8128            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8129            default_model: "gpt-5.5".to_string(),
8130            reasoning: Some(REASONING_HIGH.to_string()),
8131            auto_compact_token_limit: Some(999),
8132            model_edit_tools: std::collections::HashMap::from([(
8133                "gpt-5.5".to_string(),
8134                EDIT_TOOL_PATCH.to_string(),
8135            )]),
8136            model_parallel_tool_calls: std::collections::HashMap::from([(
8137                "gpt-5.5".to_string(),
8138                true,
8139            )]),
8140            model_profiles: std::collections::HashMap::from([(
8141                "gpt-5.5".to_string(),
8142                test_model_profile("gpt-5.5"),
8143            )]),
8144            ..RuntimeConfig::default()
8145        })
8146        .await;
8147
8148        let tool_names = request
8149            .tools
8150            .iter()
8151            .map(|tool| tool.name.as_str())
8152            .collect::<Vec<_>>();
8153        assert!(tool_names.contains(&"apply_patch"));
8154        assert!(!tool_names.contains(&"edit"));
8155        assert_eq!(request.reasoning.level.as_deref(), Some(REASONING_HIGH));
8156        assert_eq!(request.runtime.parallel_tool_calls, Some(true));
8157        assert_eq!(request.runtime.auto_compact_token_limit, Some(999));
8158    }
8159
8160    #[tokio::test]
8161    async fn runtime_tool_allowlist_filters_advertised_tools() {
8162        let request = captured_profile_request(RuntimeConfig {
8163            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8164            default_model: "gpt-5.5".to_string(),
8165            tool_allowlist: vec!["edit".to_string()],
8166            model_profiles: std::collections::HashMap::from([(
8167                "gpt-5.5".to_string(),
8168                test_model_profile("gpt-5.5"),
8169            )]),
8170            ..RuntimeConfig::default()
8171        })
8172        .await;
8173
8174        let tool_names = request
8175            .tools
8176            .iter()
8177            .map(|tool| tool.name.as_str())
8178            .collect::<Vec<_>>();
8179        assert_eq!(tool_names, vec!["edit"]);
8180    }
8181
8182    /// Builds a store-backed runtime turn for a thread created with the given per-thread
8183    /// overrides and returns the captured inference request.
8184    async fn captured_thread_override_request(
8185        runtime: &Arc<Runtime>,
8186        requests: &Arc<StdMutex<Vec<AgentInferenceRequest>>>,
8187        tool_allowlist: Vec<String>,
8188        developer_instructions: Option<String>,
8189        external_tools: Vec<ToolSpec>,
8190    ) -> AgentInferenceRequest {
8191        let thread_id = runtime
8192            .create_thread_with(CreateThreadRequest {
8193                title: Some("Thread overrides".to_string()),
8194                workspace: test_workspace(),
8195                workspace_id: None,
8196                root_id: None,
8197                provider: None,
8198                model: None,
8199                selection_mode: None,
8200                tool_allowlist,
8201                developer_instructions,
8202                external_tools,
8203                runner: None,
8204            })
8205            .await
8206            .unwrap()
8207            .thread_id;
8208        let mut rx = runtime.subscribe_events();
8209        let turn_id = runtime
8210            .start_turn(StartTurnRequest {
8211                thread_id,
8212                message: "hello".to_string(),
8213                images: Vec::new(),
8214                provider_override: None,
8215                model_override: None,
8216                reasoning_override: None,
8217                workspace: test_workspace(),
8218                instructions: crate::default_instructions(),
8219                developer_context: None,
8220                task_ledger_required: false,
8221            })
8222            .await
8223            .unwrap();
8224        tokio::time::timeout(std::time::Duration::from_secs(5), async {
8225            loop {
8226                let envelope = rx.recv().await.unwrap();
8227                if envelope.turn_id.as_deref() != Some(&turn_id) {
8228                    continue;
8229                }
8230                match envelope.event {
8231                    RoderEvent::TurnCompleted(_) => break,
8232                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8233                    _ => {}
8234                }
8235            }
8236        })
8237        .await
8238        .unwrap();
8239        requests.lock().unwrap().pop().expect("captured request")
8240    }
8241
8242    #[tokio::test]
8243    async fn thread_tool_allowlist_filters_only_that_thread() {
8244        let requests = Arc::new(StdMutex::new(Vec::new()));
8245        let thread_root =
8246            std::env::temp_dir().join(format!("roder-thread-allowlist-{}", uuid::Uuid::new_v4()));
8247        let mut builder = ExtensionRegistryBuilder::new();
8248        builder.inference_engine(Arc::new(SwitchCaptureEngine {
8249            requests: requests.clone(),
8250        }));
8251        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8252            base_path: thread_root.clone(),
8253        }));
8254        builder.tool_contributor(Arc::new(ProfileToolContributor));
8255        let runtime = Arc::new(
8256            Runtime::new(
8257                builder.build().unwrap(),
8258                RuntimeConfig {
8259                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8260                    default_model: "gpt-5.5".to_string(),
8261                    model_profiles: std::collections::HashMap::from([(
8262                        "gpt-5.5".to_string(),
8263                        test_model_profile("gpt-5.5"),
8264                    )]),
8265                    ..RuntimeConfig::default()
8266                },
8267            )
8268            .unwrap(),
8269        );
8270
8271        let allowlisted = captured_thread_override_request(
8272            &runtime,
8273            &requests,
8274            vec!["edit".to_string()],
8275            None,
8276            Vec::new(),
8277        )
8278        .await;
8279        let unrestricted =
8280            captured_thread_override_request(&runtime, &requests, Vec::new(), None, Vec::new())
8281                .await;
8282
8283        let allowlisted_names = allowlisted
8284            .tools
8285            .iter()
8286            .map(|tool| tool.name.as_str())
8287            .collect::<Vec<_>>();
8288        assert_eq!(allowlisted_names, vec!["edit"]);
8289        let unrestricted_names = unrestricted
8290            .tools
8291            .iter()
8292            .map(|tool| tool.name.as_str())
8293            .collect::<Vec<_>>();
8294        assert!(unrestricted_names.contains(&"edit"));
8295        assert!(unrestricted_names.len() > 1);
8296
8297        let _ = std::fs::remove_dir_all(thread_root);
8298    }
8299
8300    /// Builds a store-backed runtime whose `RuntimeConfig.tool_allowlist` is `["edit"]`.
8301    fn runtime_with_edit_allowlist(
8302        requests: &Arc<StdMutex<Vec<AgentInferenceRequest>>>,
8303        thread_root: &std::path::Path,
8304    ) -> Arc<Runtime> {
8305        let mut builder = ExtensionRegistryBuilder::new();
8306        builder.inference_engine(Arc::new(SwitchCaptureEngine {
8307            requests: requests.clone(),
8308        }));
8309        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8310            base_path: thread_root.to_path_buf(),
8311        }));
8312        builder.tool_contributor(Arc::new(ProfileToolContributor));
8313        Arc::new(
8314            Runtime::new(
8315                builder.build().unwrap(),
8316                RuntimeConfig {
8317                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8318                    default_model: "gpt-5.5".to_string(),
8319                    tool_allowlist: vec!["edit".to_string()],
8320                    model_profiles: std::collections::HashMap::from([(
8321                        "gpt-5.5".to_string(),
8322                        test_model_profile("gpt-5.5"),
8323                    )]),
8324                    ..RuntimeConfig::default()
8325                },
8326            )
8327            .unwrap(),
8328        )
8329    }
8330
8331    #[tokio::test]
8332    async fn runtime_and_thread_allowlists_intersect() {
8333        let requests = Arc::new(StdMutex::new(Vec::new()));
8334        let thread_root = std::env::temp_dir().join(format!(
8335            "roder-allowlist-intersect-{}",
8336            uuid::Uuid::new_v4()
8337        ));
8338        let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8339
8340        let request = captured_thread_override_request(
8341            &runtime,
8342            &requests,
8343            vec!["edit".to_string(), "write_file".to_string()],
8344            None,
8345            Vec::new(),
8346        )
8347        .await;
8348
8349        // The thread allowlist must not re-enable tools the runtime allowlist bans.
8350        let names = request
8351            .tools
8352            .iter()
8353            .map(|tool| tool.name.as_str())
8354            .collect::<Vec<_>>();
8355        assert_eq!(names, vec!["edit"]);
8356
8357        let _ = std::fs::remove_dir_all(thread_root);
8358    }
8359
8360    #[tokio::test]
8361    async fn route_tool_call_denies_tools_outside_allowlists() {
8362        let requests = Arc::new(StdMutex::new(Vec::new()));
8363        let thread_root =
8364            std::env::temp_dir().join(format!("roder-allowlist-dispatch-{}", uuid::Uuid::new_v4()));
8365        let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8366        let thread_id = runtime
8367            .create_thread_with(CreateThreadRequest {
8368                title: Some("Dispatch allowlist".to_string()),
8369                workspace: test_workspace(),
8370                workspace_id: None,
8371                root_id: None,
8372                provider: None,
8373                model: None,
8374                selection_mode: None,
8375                tool_allowlist: vec!["edit".to_string(), "write_file".to_string()],
8376                developer_instructions: None,
8377                external_tools: Vec::new(),
8378                runner: None,
8379            })
8380            .await
8381            .unwrap()
8382            .thread_id;
8383
8384        // write_file is registered and on the thread allowlist but banned by the runtime allowlist.
8385        let result = runtime
8386            .route_tool_call(
8387                &thread_id,
8388                &"turn-allowlist-dispatch".to_string(),
8389                roder_api::inference::ToolCallCompleted {
8390                    id: "call-1".to_string(),
8391                    name: "write_file".to_string(),
8392                    arguments: r#"{"path":"a.txt","content":"hi"}"#.to_string(),
8393                },
8394                None,
8395                None,
8396            )
8397            .await
8398            .unwrap();
8399
8400        assert!(result.is_error);
8401        assert!(
8402            result
8403                .result
8404                .contains("not permitted by the tool allowlist"),
8405            "unexpected result: {}",
8406            result.result
8407        );
8408
8409        let _ = std::fs::remove_dir_all(thread_root);
8410    }
8411
8412    #[tokio::test]
8413    async fn route_tool_calls_denies_agent_swarm_mixed_with_other_tools() {
8414        let requests = Arc::new(StdMutex::new(Vec::new()));
8415        let thread_root =
8416            std::env::temp_dir().join(format!("roder-swarm-exclusive-{}", uuid::Uuid::new_v4()));
8417        let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8418
8419        // A response that mixes agent_swarm with another tool is denied wholesale:
8420        // both calls get an error result with retry guidance, and no tool runs.
8421        let results = runtime
8422            .route_tool_calls(
8423                &"thread-swarm".to_string(),
8424                &"turn-swarm".to_string(),
8425                vec![
8426                    roder_api::inference::ToolCallCompleted {
8427                        id: "swarm-1".to_string(),
8428                        name: "agent_swarm".to_string(),
8429                        arguments: "{}".to_string(),
8430                    },
8431                    roder_api::inference::ToolCallCompleted {
8432                        id: "read-1".to_string(),
8433                        name: "read_file".to_string(),
8434                        arguments: "{}".to_string(),
8435                    },
8436                ],
8437                true,
8438                None,
8439                None,
8440            )
8441            .await
8442            .unwrap();
8443
8444        assert_eq!(results.len(), 2, "every tool_call_id must get a response");
8445        assert_eq!(results[0].id, "swarm-1");
8446        assert_eq!(results[1].id, "read-1");
8447        for result in &results {
8448            assert!(result.is_error);
8449            assert!(
8450                result.result.contains("only tool call"),
8451                "unexpected result: {}",
8452                result.result
8453            );
8454        }
8455
8456        let _ = std::fs::remove_dir_all(thread_root);
8457    }
8458
8459    #[tokio::test]
8460    async fn route_tool_calls_emits_agent_swarm_started_event() {
8461        let requests = Arc::new(StdMutex::new(Vec::new()));
8462        let thread_root =
8463            std::env::temp_dir().join(format!("roder-swarm-started-{}", uuid::Uuid::new_v4()));
8464        let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8465        let mut events = runtime.subscribe_events();
8466
8467        // A single agent_swarm call (the valid shape) emits AgentSwarmStarted on
8468        // the event bus before dispatch, with the child count parsed from args.
8469        let _ = runtime
8470            .route_tool_calls(
8471                &"thread-swarm".to_string(),
8472                &"turn-swarm".to_string(),
8473                vec![roder_api::inference::ToolCallCompleted {
8474                    id: "swarm-1".to_string(),
8475                    name: "agent_swarm".to_string(),
8476                    arguments: r#"{"description":"x","prompt_template":"Read {{item}}","items":["a.rs","b.rs"]}"#
8477                        .to_string(),
8478                }],
8479                true,
8480                None,
8481                None,
8482            )
8483            .await
8484            .unwrap();
8485
8486        let mut started_child_count = None;
8487        for _ in 0..16 {
8488            let envelope = tokio::time::timeout(std::time::Duration::from_secs(2), events.recv())
8489                .await
8490                .unwrap()
8491                .unwrap();
8492            if let RoderEvent::AgentSwarmStarted(event) = envelope.event {
8493                started_child_count = Some(event.child_count);
8494                assert_eq!(event.tool_id, "swarm-1");
8495                break;
8496            }
8497        }
8498        assert_eq!(started_child_count, Some(2));
8499
8500        let _ = std::fs::remove_dir_all(thread_root);
8501    }
8502
8503    #[test]
8504    fn agent_swarm_child_count_sums_items_and_resumes() {
8505        assert_eq!(
8506            agent_swarm_child_count(r#"{"description":"x","items":["a","b","c"]}"#),
8507            3
8508        );
8509        assert_eq!(
8510            agent_swarm_child_count(
8511                r#"{"description":"x","items":["a"],"resume_agent_ids":{"id1":"continue"}}"#
8512            ),
8513            2
8514        );
8515        // Malformed input is lenient.
8516        assert_eq!(agent_swarm_child_count("not json"), 0);
8517    }
8518
8519    #[test]
8520    fn parse_swarm_counts_reads_summary_with_omitted_buckets() {
8521        let text = "<agent_swarm_result>\n<summary>completed: 2, failed: 1</summary>\n</agent_swarm_result>";
8522        assert_eq!(parse_swarm_counts(text), Some((2, 1, 0)));
8523        let text = "<agent_swarm_result>\n<summary>completed: 0</summary>\n</agent_swarm_result>";
8524        assert_eq!(parse_swarm_counts(text), Some((0, 0, 0)));
8525        // Not a swarm result.
8526        assert_eq!(parse_swarm_counts("just text"), None);
8527    }
8528
8529    /// Signals when inference starts, then waits for `proceed` and fails the stream.
8530    struct SignalledFailureEngine {
8531        started: tokio::sync::mpsc::UnboundedSender<()>,
8532        proceed: Arc<tokio::sync::Notify>,
8533    }
8534
8535    #[async_trait::async_trait]
8536    impl InferenceEngine for SignalledFailureEngine {
8537        fn id(&self) -> String {
8538            roder_api::catalog::PROVIDER_MOCK.to_string()
8539        }
8540
8541        fn capabilities(&self) -> InferenceCapabilities {
8542            InferenceCapabilities::coding_agent_default()
8543        }
8544
8545        async fn list_models(
8546            &self,
8547            _ctx: InferenceProviderContext<'_>,
8548        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
8549            Ok(roder_api::catalog::models_for_provider(
8550                roder_api::catalog::PROVIDER_MOCK,
8551                true,
8552            ))
8553        }
8554
8555        async fn stream_turn(
8556            &self,
8557            _ctx: InferenceTurnContext<'_>,
8558            _request: AgentInferenceRequest,
8559        ) -> anyhow::Result<InferenceEventStream> {
8560            let _ = self.started.send(());
8561            self.proceed.notified().await;
8562            anyhow::bail!("engine failed mid-turn")
8563        }
8564    }
8565
8566    #[tokio::test]
8567    async fn failed_turn_sweeps_pending_external_tool_calls() {
8568        let (started_tx, mut started_rx) = tokio::sync::mpsc::unbounded_channel();
8569        let proceed = Arc::new(tokio::sync::Notify::new());
8570        let mut builder = ExtensionRegistryBuilder::new();
8571        builder.inference_engine(Arc::new(SignalledFailureEngine {
8572            started: started_tx,
8573            proceed: proceed.clone(),
8574        }));
8575        let runtime =
8576            Arc::new(Runtime::new(builder.build().unwrap(), RuntimeConfig::default()).unwrap());
8577        let mut rx = runtime.subscribe_events();
8578        let turn_id = runtime
8579            .start_turn(StartTurnRequest {
8580                thread_id: "thread-sweep".to_string(),
8581                message: "go".to_string(),
8582                images: Vec::new(),
8583                provider_override: None,
8584                model_override: None,
8585                reasoning_override: None,
8586                workspace: test_workspace(),
8587                instructions: InstructionBundle {
8588                    system: None,
8589                    developer: None,
8590                    developer_context: None,
8591                },
8592                developer_context: None,
8593                task_ledger_required: false,
8594            })
8595            .await
8596            .unwrap();
8597        tokio::time::timeout(std::time::Duration::from_secs(5), started_rx.recv())
8598            .await
8599            .unwrap()
8600            .unwrap();
8601
8602        let (tx, _pending_rx) = oneshot::channel();
8603        runtime.pending_external_tool_calls.lock().await.insert(
8604            "exttool-sweep-test".to_string(),
8605            PendingExternalToolCall {
8606                thread_id: "thread-sweep".to_string(),
8607                turn_id: turn_id.clone(),
8608                tool_id: "call-1".to_string(),
8609                tool_name: "acme_lookup".to_string(),
8610                tx,
8611            },
8612        );
8613        proceed.notify_one();
8614
8615        let outcome = tokio::time::timeout(std::time::Duration::from_secs(5), async {
8616            loop {
8617                let envelope = rx.recv().await.unwrap();
8618                if let RoderEvent::ExternalToolCallResolved(event) = envelope.event
8619                    && event.request_id == "exttool-sweep-test"
8620                {
8621                    break event.outcome;
8622                }
8623            }
8624        })
8625        .await
8626        .expect("turn failure must resolve pending external tool calls");
8627        assert_eq!(outcome, ExternalToolCallOutcome::Cancelled);
8628        assert!(runtime.pending_external_tool_calls.lock().await.is_empty());
8629    }
8630
8631    #[tokio::test]
8632    async fn thread_developer_instructions_layer_under_harness_prompt() {
8633        let requests = Arc::new(StdMutex::new(Vec::new()));
8634        let thread_root = std::env::temp_dir().join(format!(
8635            "roder-thread-instructions-{}",
8636            uuid::Uuid::new_v4()
8637        ));
8638        let mut builder = ExtensionRegistryBuilder::new();
8639        builder.inference_engine(Arc::new(SwitchCaptureEngine {
8640            requests: requests.clone(),
8641        }));
8642        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8643            base_path: thread_root.clone(),
8644        }));
8645        builder.tool_contributor(Arc::new(ProfileToolContributor));
8646        let runtime =
8647            Arc::new(Runtime::new(builder.build().unwrap(), RuntimeConfig::default()).unwrap());
8648
8649        let request = captured_thread_override_request(
8650            &runtime,
8651            &requests,
8652            Vec::new(),
8653            Some("You are embedded in a host app.".to_string()),
8654            Vec::new(),
8655        )
8656        .await;
8657
8658        let system = request.instructions.system.expect("system instructions");
8659        assert!(system.starts_with("You are Roder"));
8660        let developer = request
8661            .instructions
8662            .developer
8663            .expect("developer instructions");
8664        assert!(developer.starts_with("You are embedded in a host app."));
8665
8666        let plain =
8667            captured_thread_override_request(&runtime, &requests, Vec::new(), None, Vec::new())
8668                .await;
8669        assert_eq!(plain.instructions.developer, None);
8670
8671        let _ = std::fs::remove_dir_all(thread_root);
8672    }
8673
8674    #[tokio::test]
8675    async fn thread_external_tools_are_advertised_and_shadow_builtins() {
8676        let requests = Arc::new(StdMutex::new(Vec::new()));
8677        let thread_root = std::env::temp_dir().join(format!(
8678            "roder-thread-external-tools-{}",
8679            uuid::Uuid::new_v4()
8680        ));
8681        let mut builder = ExtensionRegistryBuilder::new();
8682        builder.inference_engine(Arc::new(SwitchCaptureEngine {
8683            requests: requests.clone(),
8684        }));
8685        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8686            base_path: thread_root.clone(),
8687        }));
8688        builder.tool_contributor(Arc::new(ProfileToolContributor));
8689        let runtime =
8690            Arc::new(Runtime::new(builder.build().unwrap(), RuntimeConfig::default()).unwrap());
8691
8692        let external_tools = vec![
8693            ToolSpec {
8694                name: "acme_lookup".to_string(),
8695                description: "Look up Acme workspace state.".to_string(),
8696                parameters: serde_json::json!({
8697                    "type": "object",
8698                    "properties": { "query": { "type": "string" } },
8699                    "required": ["query"]
8700                }),
8701            },
8702            ToolSpec {
8703                name: "edit".to_string(),
8704                description: "Host-managed edit.".to_string(),
8705                parameters: serde_json::json!({ "type": "object" }),
8706            },
8707        ];
8708        let request =
8709            captured_thread_override_request(&runtime, &requests, Vec::new(), None, external_tools)
8710                .await;
8711
8712        let acme = request
8713            .tools
8714            .iter()
8715            .find(|tool| tool.name == "acme_lookup")
8716            .expect("external tool advertised");
8717        assert_eq!(acme.description, "Look up Acme workspace state.");
8718        assert_eq!(acme.parameters["required"][0], "query");
8719        let edits = request
8720            .tools
8721            .iter()
8722            .filter(|tool| tool.name == "edit")
8723            .collect::<Vec<_>>();
8724        assert_eq!(edits.len(), 1, "external edit shadows the builtin");
8725        assert_eq!(edits[0].description, "Host-managed edit.");
8726
8727        let plain =
8728            captured_thread_override_request(&runtime, &requests, Vec::new(), None, Vec::new())
8729                .await;
8730        assert!(plain.tools.iter().all(|tool| tool.name != "acme_lookup"));
8731        let plain_edit = plain
8732            .tools
8733            .iter()
8734            .find(|tool| tool.name == "edit")
8735            .expect("builtin edit advertised on plain thread");
8736        assert_eq!(plain_edit.description, "edit test tool");
8737
8738        let _ = std::fs::remove_dir_all(thread_root);
8739    }
8740
8741    #[tokio::test]
8742    async fn model_switch_injects_summary_and_records_profile_segments() {
8743        let requests = Arc::new(StdMutex::new(Vec::new()));
8744        let thread_root = std::env::temp_dir().join(format!(
8745            "roder-model-switch-thread-{}",
8746            uuid::Uuid::new_v4()
8747        ));
8748        let mut builder = ExtensionRegistryBuilder::new();
8749        builder.inference_engine(Arc::new(SwitchCaptureEngine {
8750            requests: requests.clone(),
8751        }));
8752        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8753            base_path: thread_root.clone(),
8754        }));
8755        builder.tool_contributor(Arc::new(ProfileToolContributor));
8756        let mut claude_profile = test_model_profile("claude-haiku-4-5-20251001");
8757        claude_profile.provider_family = ProviderFamily::Anthropic;
8758        claude_profile.edit_tool = Some(EDIT_TOOL_EDIT.to_string());
8759        let runtime = Arc::new(
8760            Runtime::new(
8761                builder.build().unwrap(),
8762                RuntimeConfig {
8763                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8764                    default_model: "gpt-5.5".to_string(),
8765                    model_profiles: std::collections::HashMap::from([
8766                        ("gpt-5.5".to_string(), test_model_profile("gpt-5.5")),
8767                        ("claude-haiku-4-5-20251001".to_string(), claude_profile),
8768                    ]),
8769                    ..RuntimeConfig::default()
8770                },
8771            )
8772            .unwrap(),
8773        );
8774        let thread_id = runtime
8775            .create_thread_with(CreateThreadRequest {
8776                title: Some("Model switch".to_string()),
8777                workspace: test_workspace(),
8778                workspace_id: None,
8779                root_id: None,
8780                provider: None,
8781                model: None,
8782                selection_mode: None,
8783                tool_allowlist: Vec::new(),
8784                developer_instructions: None,
8785                external_tools: Vec::new(),
8786                runner: None,
8787            })
8788            .await
8789            .unwrap()
8790            .thread_id;
8791        let mut rx = runtime.subscribe_events();
8792        for (message, model_override) in [
8793            ("first turn", None),
8794            ("second turn", Some("claude-haiku-4-5-20251001".to_string())),
8795        ] {
8796            let turn_id = runtime
8797                .start_turn(StartTurnRequest {
8798                    thread_id: thread_id.clone(),
8799                    message: message.to_string(),
8800                    images: Vec::new(),
8801                    provider_override: None,
8802                    model_override,
8803                    reasoning_override: None,
8804                    workspace: test_workspace(),
8805                    instructions: InstructionBundle::default(),
8806                    developer_context: None,
8807                    task_ledger_required: false,
8808                })
8809                .await
8810                .unwrap();
8811            tokio::time::timeout(std::time::Duration::from_secs(5), async {
8812                loop {
8813                    let envelope = rx.recv().await.unwrap();
8814                    if envelope.turn_id.as_deref() != Some(&turn_id) {
8815                        continue;
8816                    }
8817                    match envelope.event {
8818                        RoderEvent::TurnCompleted(_) => break,
8819                        RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8820                        _ => {}
8821                    }
8822                }
8823            })
8824            .await
8825            .unwrap();
8826        }
8827
8828        let captured = requests.lock().unwrap().clone();
8829        assert_eq!(captured.len(), 2);
8830        assert!(captured[1].transcript.iter().any(|item| {
8831            matches!(
8832                item,
8833                TranscriptItem::UserMessage(message)
8834                    if message.text.starts_with(MODEL_SWITCH_SUMMARY_PREFIX)
8835                        && message.text.contains("previous profile mock/gpt-5.5")
8836                        && message.text.contains("Current profile mock/claude-haiku-4-5-20251001")
8837                        && message.text.contains("Available tools now:")
8838            )
8839        }));
8840
8841        let snapshot = runtime
8842            .thread_store
8843            .as_ref()
8844            .unwrap()
8845            .load_thread(&thread_id)
8846            .await
8847            .unwrap()
8848            .unwrap();
8849        let trace_segments = snapshot
8850            .turns
8851            .iter()
8852            .flat_map(|turn| &turn.items)
8853            .filter(|item| {
8854                matches!(
8855                    item,
8856                    TranscriptItem::ProviderMetadata(value)
8857                        if value.get("kind").and_then(serde_json::Value::as_str)
8858                            == Some(MODEL_PROFILE_TRACE_KIND)
8859                            && value.get("segment").and_then(serde_json::Value::as_str)
8860                                == Some("assistant")
8861                )
8862            })
8863            .count();
8864        assert!(trace_segments >= 2);
8865        let _ = std::fs::remove_dir_all(thread_root);
8866    }
8867
8868    struct CountingTaskTool {
8869        calls: Arc<StdMutex<u32>>,
8870    }
8871
8872    #[async_trait::async_trait]
8873    impl ToolExecutor for CountingTaskTool {
8874        fn spec(&self) -> ToolSpec {
8875            ToolSpec {
8876                name: "task".to_string(),
8877                description: "Dispatch a test subagent.".to_string(),
8878                parameters: serde_json::json!({
8879                    "type": "object",
8880                    "properties": {
8881                        "description": { "type": "string" },
8882                        "prompt": { "type": "string" },
8883                        "parent_deadline_seconds": { "type": "integer" }
8884                    },
8885                    "required": ["description", "prompt"],
8886                    "additionalProperties": false
8887                }),
8888            }
8889        }
8890
8891        async fn execute(
8892            &self,
8893            _ctx: ToolExecutionContext,
8894            call: ToolCall,
8895        ) -> anyhow::Result<ToolResult> {
8896            *self.calls.lock().unwrap() += 1;
8897            Ok(ToolResult {
8898                id: call.id,
8899                name: call.name,
8900                text: "started child".to_string(),
8901                data: serde_json::json!({}),
8902                is_error: false,
8903            })
8904        }
8905    }
8906
8907    #[tokio::test]
8908    async fn runtime_profile_reaches_inference_request_and_turn_metadata() {
8909        let captured = Arc::new(StdMutex::new(None));
8910        let mut builder = ExtensionRegistryBuilder::new();
8911        builder.inference_engine(Arc::new(CapturingEngine {
8912            request: captured.clone(),
8913        }));
8914        let runtime = Arc::new(
8915            Runtime::new(
8916                builder.build().unwrap(),
8917                RuntimeConfig {
8918                    runtime_profile: RuntimeProfile::NonInteractive,
8919                    ..RuntimeConfig::default()
8920                },
8921            )
8922            .unwrap(),
8923        );
8924        let mut rx = runtime.subscribe_events();
8925        let turn_id = runtime
8926            .start_turn(StartTurnRequest {
8927                thread_id: "thread-profile".to_string(),
8928                message: "work unattended".to_string(),
8929                images: Vec::new(),
8930                provider_override: None,
8931                model_override: None,
8932                reasoning_override: None,
8933                workspace: test_workspace(),
8934                instructions: InstructionBundle {
8935                    system: None,
8936                    developer: Some("base developer".to_string()),
8937                    developer_context: None,
8938                },
8939                developer_context: None,
8940                task_ledger_required: false,
8941            })
8942            .await
8943            .unwrap();
8944
8945        let mut observed_profile = None;
8946        tokio::time::timeout(std::time::Duration::from_secs(5), async {
8947            loop {
8948                let envelope = rx.recv().await.unwrap();
8949                if envelope.turn_id.as_deref() != Some(&turn_id) {
8950                    continue;
8951                }
8952                match envelope.event {
8953                    RoderEvent::TurnStarted(event) => {
8954                        observed_profile = Some(event.runtime_profile);
8955                    }
8956                    RoderEvent::TurnCompleted(_) => break,
8957                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8958                    _ => {}
8959                }
8960            }
8961        })
8962        .await
8963        .unwrap();
8964
8965        assert_eq!(observed_profile, Some(RuntimeProfile::NonInteractive));
8966        let request = captured.lock().unwrap().clone().unwrap();
8967        assert_eq!(request.runtime.profile, RuntimeProfile::NonInteractive);
8968        let developer = request.instructions.developer.unwrap();
8969        assert!(developer.contains("base developer"));
8970        assert!(developer.contains("non-interactive profile"));
8971    }
8972
8973    #[tokio::test]
8974    async fn global_policy_mode_changes_do_not_create_runtime_thread_directory() {
8975        let workspace = runtime_test_workspace("global-policy-mode");
8976        let thread_root = workspace.join("threads");
8977        let mut builder = ExtensionRegistryBuilder::new();
8978        builder.inference_engine(Arc::new(FakeInferenceEngine));
8979        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8980            base_path: thread_root.clone(),
8981        }));
8982        let runtime = Runtime::new(
8983            builder.build().unwrap(),
8984            RuntimeConfig {
8985                workspace: Some(workspace.display().to_string()),
8986                ..Default::default()
8987            },
8988        )
8989        .unwrap();
8990
8991        runtime
8992            .set_policy_mode(PolicyMode::AcceptAll, Some("test".to_string()))
8993            .await
8994            .unwrap();
8995
8996        assert!(!thread_root.join("runtime").exists());
8997        let _ = std::fs::remove_dir_all(workspace);
8998    }
8999
9000    #[tokio::test]
9001    async fn task_ledger_enforcement_injects_eval_reminder_before_work() {
9002        let captured = Arc::new(StdMutex::new(None));
9003        let mut builder = ExtensionRegistryBuilder::new();
9004        builder.inference_engine(Arc::new(CapturingEngine {
9005            request: captured.clone(),
9006        }));
9007        builder.tool_contributor(Arc::new(
9008            roder_ext_task_ledger::TaskLedgerToolContributor::default(),
9009        ));
9010        let runtime = Arc::new(
9011            Runtime::new(
9012                builder.build().unwrap(),
9013                RuntimeConfig {
9014                    runtime_profile: RuntimeProfile::Eval,
9015                    policy_mode: PolicyMode::Bypass,
9016                    agent_swarm_mode: false,
9017                    ..RuntimeConfig::default()
9018                },
9019            )
9020            .unwrap(),
9021        );
9022        let mut rx = runtime.subscribe_events();
9023        let turn_id = runtime
9024            .start_turn(StartTurnRequest {
9025                thread_id: "thread-ledger".to_string(),
9026                message: "decomposed work".to_string(),
9027                images: Vec::new(),
9028                provider_override: None,
9029                model_override: None,
9030                reasoning_override: None,
9031                workspace: test_workspace(),
9032                instructions: InstructionBundle::default(),
9033                developer_context: None,
9034                task_ledger_required: true,
9035            })
9036            .await
9037            .unwrap();
9038
9039        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9040            loop {
9041                let envelope = rx.recv().await.unwrap();
9042                if envelope.turn_id.as_deref() == Some(&turn_id)
9043                    && matches!(envelope.event, RoderEvent::TurnCompleted(_))
9044                {
9045                    break;
9046                }
9047            }
9048        })
9049        .await
9050        .unwrap();
9051
9052        let request = captured.lock().unwrap().clone().unwrap();
9053        let developer = request.instructions.developer.unwrap();
9054        assert!(developer.contains("Task Ledger Required"));
9055        assert!(developer.contains("task_ledger.update"));
9056        let tool_names: Vec<_> = request
9057            .tools
9058            .iter()
9059            .map(|tool| tool.name.as_str())
9060            .collect();
9061        assert!(
9062            tool_names.contains(&TASK_LEDGER_TOOL_NAME),
9063            "tool names: {tool_names:?}"
9064        );
9065        assert_eq!(
9066            request.tool_choice,
9067            ToolChoice::Specific(TASK_LEDGER_TOOL_NAME.to_string())
9068        );
9069        assert_eq!(request.tools.len(), 1);
9070        assert_eq!(request.tools[0].name, TASK_LEDGER_TOOL_NAME);
9071    }
9072
9073    #[tokio::test]
9074    async fn eval_task_ledger_blocks_final_answer_until_open_items_are_completed() {
9075        let requests = Arc::new(StdMutex::new(Vec::new()));
9076        let mut builder = ExtensionRegistryBuilder::new();
9077        builder.inference_engine(Arc::new(TaskLedgerCompletionGateEngine {
9078            calls: StdMutex::new(0),
9079            requests: requests.clone(),
9080        }));
9081        builder.tool_contributor(Arc::new(
9082            roder_ext_task_ledger::TaskLedgerToolContributor::default(),
9083        ));
9084        let runtime = Arc::new(
9085            Runtime::new(
9086                builder.build().unwrap(),
9087                RuntimeConfig {
9088                    runtime_profile: RuntimeProfile::Eval,
9089                    policy_mode: PolicyMode::Bypass,
9090                    agent_swarm_mode: false,
9091                    ..RuntimeConfig::default()
9092                },
9093            )
9094            .unwrap(),
9095        );
9096        let mut rx = runtime.subscribe_events();
9097        let turn_id = runtime
9098            .start_turn(StartTurnRequest {
9099                thread_id: "thread-ledger-completion".to_string(),
9100                message: "write the answer file".to_string(),
9101                images: Vec::new(),
9102                provider_override: None,
9103                model_override: None,
9104                reasoning_override: None,
9105                workspace: test_workspace(),
9106                instructions: InstructionBundle::default(),
9107                developer_context: None,
9108                task_ledger_required: true,
9109            })
9110            .await
9111            .unwrap();
9112
9113        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9114            loop {
9115                let envelope = rx.recv().await.unwrap();
9116                if envelope.turn_id.as_deref() != Some(&turn_id) {
9117                    continue;
9118                }
9119                match envelope.event {
9120                    RoderEvent::TurnCompleted(_) => break,
9121                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9122                    _ => {}
9123                }
9124            }
9125        })
9126        .await
9127        .unwrap();
9128
9129        let requests = requests.lock().unwrap().clone();
9130        assert_eq!(requests.len(), 4);
9131        assert!(requests[2].transcript.iter().any(|item| {
9132            matches!(
9133                item,
9134                TranscriptItem::UserMessage(message)
9135                    if message.text.contains("Task Ledger Completion Required")
9136                        && message.text.contains("Write /app/result.txt")
9137            )
9138        }));
9139        assert!(requests[3].transcript.iter().any(|item| {
9140            matches!(
9141                item,
9142                TranscriptItem::ToolResult(result)
9143                    if result.name.as_deref() == Some(TASK_LEDGER_TOOL_NAME)
9144                        && result.result.contains("Task ledger: 2/2 completed")
9145            )
9146        }));
9147    }
9148
9149    #[tokio::test]
9150    async fn eval_task_ledger_checkpoint_requests_scoreable_file_before_final_reserve() {
9151        let requests = Arc::new(StdMutex::new(Vec::new()));
9152        let mut builder = ExtensionRegistryBuilder::new();
9153        builder.inference_engine(Arc::new(TaskLedgerCompletionGateEngine {
9154            calls: StdMutex::new(0),
9155            requests: requests.clone(),
9156        }));
9157        builder.tool_contributor(Arc::new(
9158            roder_ext_task_ledger::TaskLedgerToolContributor::default(),
9159        ));
9160        let runtime = Arc::new(
9161            Runtime::new(
9162                builder.build().unwrap(),
9163                RuntimeConfig {
9164                    runtime_profile: RuntimeProfile::Eval,
9165                    policy_mode: PolicyMode::Bypass,
9166                    agent_swarm_mode: false,
9167                    turn_deadline_seconds: Some(120),
9168                    ..RuntimeConfig::default()
9169                },
9170            )
9171            .unwrap(),
9172        );
9173        let mut rx = runtime.subscribe_events();
9174        let turn_id = runtime
9175            .start_turn(StartTurnRequest {
9176                thread_id: "thread-ledger-checkpoint".to_string(),
9177                message: "write the answer file".to_string(),
9178                images: Vec::new(),
9179                provider_override: None,
9180                model_override: None,
9181                reasoning_override: None,
9182                workspace: test_workspace(),
9183                instructions: InstructionBundle::default(),
9184                developer_context: None,
9185                task_ledger_required: true,
9186            })
9187            .await
9188            .unwrap();
9189
9190        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9191            loop {
9192                let envelope = rx.recv().await.unwrap();
9193                if envelope.turn_id.as_deref() != Some(&turn_id) {
9194                    continue;
9195                }
9196                match envelope.event {
9197                    RoderEvent::TurnCompleted(_) => break,
9198                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9199                    _ => {}
9200                }
9201            }
9202        })
9203        .await
9204        .unwrap();
9205
9206        let requests = requests.lock().unwrap().clone();
9207        assert!(requests.len() >= 2);
9208        assert!(requests[1].transcript.iter().any(|item| {
9209            matches!(
9210                item,
9211                TranscriptItem::UserMessage(message)
9212                    if message.text.contains("Scoreable Output Checkpoint")
9213                        && message.text.contains("ensure the required output file(s) exist")
9214                        && message.text.contains("Write /app/result.txt")
9215            )
9216        }));
9217    }
9218
9219    #[test]
9220    fn deadline_task_ledger_prompt_preserves_scoreable_work_instruction() {
9221        let prompt = task_ledger_deadline_completion_prompt(
9222            12,
9223            30,
9224            "Task Ledger Completion Required: write /app/result.txt, then call task_ledger.update",
9225        );
9226
9227        assert!(prompt.contains("12 seconds remain"));
9228        assert!(prompt.contains("create or update the required scoreable output files"));
9229        assert!(prompt.contains("write /app/result.txt"));
9230        assert!(prompt.contains(TASK_LEDGER_TOOL_NAME));
9231    }
9232
9233    #[test]
9234    fn scoreable_checkpoint_prompt_preserves_provisional_file_instruction() {
9235        let prompt = task_ledger_scoreable_checkpoint_prompt(
9236            120,
9237            "Task Ledger Completion Required: write /app/result.txt, then call task_ledger.update",
9238        );
9239
9240        assert!(prompt.contains("120 seconds remain"));
9241        assert!(prompt.contains("best evidence-backed answer"));
9242        assert!(prompt.contains("even if provisional"));
9243        assert!(prompt.contains("preserve that candidate"));
9244        assert!(prompt.contains("partial-coverage"));
9245        assert!(prompt.contains("write /app/result.txt"));
9246        assert!(prompt.contains(TASK_LEDGER_TOOL_NAME));
9247    }
9248
9249    #[test]
9250    fn open_task_ledger_moves_inference_timeout_to_scoreable_checkpoint() {
9251        let deadline = Some(OffsetDateTime::now_utc() + Duration::seconds(870));
9252        let transcript = vec![TranscriptItem::ToolResult(ToolResultRecord {
9253            id: "ledger-open".to_string(),
9254            name: Some(TASK_LEDGER_TOOL_NAME.to_string()),
9255            result: "Task ledger: 0/1 completed\n- pending: Write /app/result.txt [write]"
9256                .to_string(),
9257            display_payload: None,
9258            is_error: false,
9259        })];
9260
9261        let (_, action) = inference_timeout_deadline(
9262            deadline,
9263            RuntimeProfile::Eval,
9264            true,
9265            30,
9266            false,
9267            0,
9268            &transcript,
9269        )
9270        .unwrap();
9271
9272        assert_eq!(action, InferenceTimeoutAction::ScoreableCheckpoint);
9273    }
9274
9275    #[tokio::test]
9276    async fn verification_gate_forces_eval_code_changes_through_review() {
9277        let mut builder = ExtensionRegistryBuilder::new();
9278        builder.inference_engine(Arc::new(VerificationGateEngine {
9279            calls: StdMutex::new(0),
9280        }));
9281        builder.tool_contributor(Arc::new(WriteFileContributor));
9282        builder.tool_contributor(Arc::new(
9283            roder_ext_verification::VerificationToolContributor,
9284        ));
9285        let runtime = Arc::new(
9286            Runtime::new(
9287                builder.build().unwrap(),
9288                RuntimeConfig {
9289                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9290                    default_model: "mock".to_string(),
9291                    runtime_profile: RuntimeProfile::Eval,
9292                    policy_mode: PolicyMode::Bypass,
9293                    agent_swarm_mode: false,
9294                    ..RuntimeConfig::default()
9295                },
9296            )
9297            .unwrap(),
9298        );
9299        let mut rx = runtime.subscribe_events();
9300        let turn_id = runtime
9301            .start_turn(StartTurnRequest {
9302                thread_id: "thread-verification".to_string(),
9303                message: "write code".to_string(),
9304                images: Vec::new(),
9305                provider_override: None,
9306                model_override: None,
9307                reasoning_override: None,
9308                workspace: test_workspace(),
9309                instructions: InstructionBundle::default(),
9310                developer_context: None,
9311                task_ledger_required: false,
9312            })
9313            .await
9314            .unwrap();
9315
9316        let mut saw_required = false;
9317        let mut saw_completed = false;
9318        let mut final_text = String::new();
9319        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9320            loop {
9321                let envelope = rx.recv().await.unwrap();
9322                if envelope.turn_id.as_deref() != Some(&turn_id) {
9323                    continue;
9324                }
9325                match envelope.event {
9326                    RoderEvent::VerificationRequired(event) => {
9327                        saw_required = true;
9328                        assert_eq!(event.changed_files, vec!["src/lib.rs"]);
9329                    }
9330                    RoderEvent::VerificationCompleted(event) => {
9331                        saw_completed = true;
9332                        assert!(event.passed);
9333                    }
9334                    RoderEvent::InferenceEventReceived(event) => {
9335                        if let InferenceEvent::MessageDelta(delta) = event.event {
9336                            final_text.push_str(&delta.text);
9337                        }
9338                    }
9339                    RoderEvent::TurnCompleted(_) => break,
9340                    _ => {}
9341                }
9342            }
9343        })
9344        .await
9345        .unwrap();
9346
9347        assert!(saw_required);
9348        assert!(saw_completed);
9349        assert!(final_text.contains("verified final"));
9350    }
9351
9352    #[tokio::test]
9353    async fn speed_policy_changes_reasoning_across_eval_model_calls_without_model_switch() {
9354        let requests = Arc::new(StdMutex::new(Vec::new()));
9355        let mut builder = ExtensionRegistryBuilder::new();
9356        builder.inference_engine(Arc::new(SpeedPolicyEngine {
9357            calls: StdMutex::new(0),
9358            requests: requests.clone(),
9359        }));
9360        builder.tool_contributor(Arc::new(WriteFileContributor));
9361        builder.tool_contributor(Arc::new(
9362            roder_ext_verification::VerificationToolContributor,
9363        ));
9364        let runtime = Arc::new(
9365            Runtime::new(
9366                builder.build().unwrap(),
9367                RuntimeConfig {
9368                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9369                    default_model: "gpt-5.5".to_string(),
9370                    runtime_profile: RuntimeProfile::Eval,
9371                    policy_mode: PolicyMode::Bypass,
9372                    agent_swarm_mode: false,
9373                    ..RuntimeConfig::default()
9374                },
9375            )
9376            .unwrap(),
9377        );
9378        let mut rx = runtime.subscribe_events();
9379        let turn_id = runtime
9380            .start_turn(StartTurnRequest {
9381                thread_id: "thread-speed-policy".to_string(),
9382                message: "write code".to_string(),
9383                images: Vec::new(),
9384                provider_override: None,
9385                model_override: None,
9386                reasoning_override: None,
9387                workspace: test_workspace(),
9388                instructions: InstructionBundle::default(),
9389                developer_context: None,
9390                task_ledger_required: false,
9391            })
9392            .await
9393            .unwrap();
9394
9395        let mut saw_speed_policy_event = false;
9396        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9397            loop {
9398                let envelope = rx.recv().await.unwrap();
9399                if envelope.turn_id.as_deref() != Some(&turn_id) {
9400                    continue;
9401                }
9402                match envelope.event {
9403                    RoderEvent::InferenceStarted(event) => {
9404                        if event.speed_policy.is_some() {
9405                            saw_speed_policy_event = true;
9406                        }
9407                    }
9408                    RoderEvent::TurnCompleted(_) => break,
9409                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9410                    _ => {}
9411                }
9412            }
9413        })
9414        .await
9415        .unwrap();
9416
9417        let requests = requests.lock().unwrap().clone();
9418        assert!(saw_speed_policy_event);
9419        assert!(requests.len() >= 4);
9420        assert!(requests.iter().all(|request| {
9421            request.model.provider == roder_api::catalog::PROVIDER_MOCK
9422                && request.model.model == "gpt-5.5"
9423        }));
9424        assert_eq!(
9425            requests[0].runtime.speed_policy.as_ref().map(|d| d.phase),
9426            Some(roder_api::inference::SpeedPolicyPhase::Orientation)
9427        );
9428        assert_eq!(requests[0].reasoning.level.as_deref(), Some(REASONING_HIGH));
9429        assert_eq!(
9430            requests[1].runtime.speed_policy.as_ref().map(|d| d.phase),
9431            Some(roder_api::inference::SpeedPolicyPhase::Execution)
9432        );
9433        assert_eq!(requests[1].reasoning.level.as_deref(), Some(REASONING_LOW));
9434        assert_eq!(
9435            requests[2].runtime.speed_policy.as_ref().map(|d| d.phase),
9436            Some(roder_api::inference::SpeedPolicyPhase::Verification)
9437        );
9438        assert_eq!(requests[2].reasoning.level.as_deref(), Some(REASONING_HIGH));
9439        assert_eq!(
9440            requests[2]
9441                .metadata
9442                .pointer("/speedPolicy/phase")
9443                .and_then(serde_json::Value::as_str),
9444            Some("verification")
9445        );
9446    }
9447
9448    #[tokio::test]
9449    async fn deadline_turn_timeout_emits_partial_result_and_clears_active_turn() {
9450        let mut builder = ExtensionRegistryBuilder::new();
9451        builder.inference_engine(Arc::new(DeadlineEngine));
9452        let runtime = Arc::new(
9453            Runtime::new(
9454                builder.build().unwrap(),
9455                RuntimeConfig {
9456                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9457                    default_model: "mock".to_string(),
9458                    runtime_profile: RuntimeProfile::Eval,
9459                    turn_deadline_seconds: Some(1),
9460                    ..RuntimeConfig::default()
9461                },
9462            )
9463            .unwrap(),
9464        );
9465        let mut rx = runtime.subscribe_events();
9466        let turn_id = runtime
9467            .start_turn(StartTurnRequest {
9468                thread_id: "thread-deadline".to_string(),
9469                message: "slow work".to_string(),
9470                images: Vec::new(),
9471                provider_override: None,
9472                model_override: None,
9473                reasoning_override: None,
9474                workspace: test_workspace(),
9475                instructions: InstructionBundle::default(),
9476                developer_context: None,
9477                task_ledger_required: false,
9478            })
9479            .await
9480            .unwrap();
9481
9482        let mut saw_partial = false;
9483        let mut saw_deadline = false;
9484        let mut failed_kind = None;
9485        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9486            loop {
9487                let envelope = rx.recv().await.unwrap();
9488                if envelope.turn_id.as_deref() != Some(&turn_id) {
9489                    continue;
9490                }
9491                match envelope.event {
9492                    RoderEvent::TurnPartialResult(event) => {
9493                        saw_partial = event.summary.contains("partial turn state");
9494                    }
9495                    RoderEvent::TurnDeadlineExceeded(event) => {
9496                        saw_deadline = event.partial_result.contains("transcript items");
9497                    }
9498                    RoderEvent::TurnFailed(event) => {
9499                        failed_kind = event.error_kind;
9500                        break;
9501                    }
9502                    _ => {}
9503                }
9504            }
9505        })
9506        .await
9507        .unwrap();
9508
9509        for _ in 0..20 {
9510            if !runtime.active_turns.read().await.contains_key(&turn_id) {
9511                break;
9512            }
9513            tokio::time::sleep(std::time::Duration::from_millis(10)).await;
9514        }
9515        assert!(saw_partial);
9516        assert!(saw_deadline);
9517        assert_eq!(failed_kind.as_deref(), Some("deadline_timeout"));
9518        assert!(!runtime.active_turns.read().await.contains_key(&turn_id));
9519    }
9520
9521    #[tokio::test]
9522    async fn deadline_skips_subagent_task_when_remaining_budget_is_too_low() {
9523        let calls = Arc::new(StdMutex::new(0));
9524        let mut builder = ExtensionRegistryBuilder::new();
9525        builder.inference_engine(Arc::new(CapturingEngine {
9526            request: Arc::new(StdMutex::new(None)),
9527        }));
9528        let task_tool = Arc::new(CountingTaskTool {
9529            calls: calls.clone(),
9530        });
9531        builder.tool_contributor(Arc::new(TestToolContributor { tool: task_tool }));
9532        let runtime = Arc::new(
9533            Runtime::new(
9534                builder.build().unwrap(),
9535                RuntimeConfig {
9536                    policy_mode: PolicyMode::Bypass,
9537                    agent_swarm_mode: false,
9538                    ..RuntimeConfig::default()
9539                },
9540            )
9541            .unwrap(),
9542        );
9543
9544        let result = runtime
9545            .route_tool_call(
9546                &"thread-deadline-task".to_string(),
9547                &"turn-deadline-task".to_string(),
9548                ToolCallCompleted {
9549                    id: "task-1".to_string(),
9550                    name: "task".to_string(),
9551                    arguments: serde_json::json!({
9552                        "description": "inspect",
9553                        "prompt": "read"
9554                    })
9555                    .to_string(),
9556                },
9557                None,
9558                Some(OffsetDateTime::now_utc() + Duration::seconds(1)),
9559            )
9560            .await
9561            .unwrap();
9562
9563        assert!(result.is_error);
9564        assert!(result.result.contains("deadline policy skipped"));
9565        assert_eq!(*calls.lock().unwrap(), 0);
9566    }
9567
9568    struct TestToolContributor {
9569        tool: Arc<dyn ToolExecutor>,
9570    }
9571
9572    impl ToolContributor for TestToolContributor {
9573        fn id(&self) -> String {
9574            "test-tool".to_string()
9575        }
9576
9577        fn contribute(&self, registry: &mut ToolRegistry) -> anyhow::Result<()> {
9578            registry.register(self.tool.clone())
9579        }
9580    }
9581
9582    #[tokio::test]
9583    async fn agent_swarm_mode_override_is_per_thread() {
9584        let runtime = Runtime::fake().unwrap();
9585        let trigger = roder_api::subagents::AgentSwarmModeTrigger::Manual;
9586
9587        // Off everywhere by default.
9588        assert!(
9589            !runtime
9590                .effective_agent_swarm_mode_for_thread("thread-a")
9591                .await
9592        );
9593        assert!(
9594            !runtime
9595                .effective_agent_swarm_mode_for_thread("thread-b")
9596                .await
9597        );
9598
9599        // Enabling on thread-a does not leak into thread-b.
9600        assert!(
9601            runtime
9602                .set_agent_swarm_mode_for_thread("thread-a", true, trigger)
9603                .await
9604        );
9605        assert!(
9606            runtime
9607                .effective_agent_swarm_mode_for_thread("thread-a")
9608                .await
9609        );
9610        assert!(
9611            !runtime
9612                .effective_agent_swarm_mode_for_thread("thread-b")
9613                .await
9614        );
9615
9616        // A per-thread `off` override wins over a runtime-global `on` default.
9617        runtime.set_agent_swarm_mode(true, trigger).await.unwrap();
9618        runtime
9619            .set_agent_swarm_mode_for_thread("thread-b", false, trigger)
9620            .await;
9621        assert!(
9622            !runtime
9623                .effective_agent_swarm_mode_for_thread("thread-b")
9624                .await,
9625            "explicit per-thread off overrides the global on default"
9626        );
9627        // A thread with no override still follows the global default.
9628        assert!(
9629            runtime
9630                .effective_agent_swarm_mode_for_thread("thread-c")
9631                .await,
9632            "threads without an override follow the runtime-global default"
9633        );
9634    }
9635
9636    #[tokio::test]
9637    async fn set_agent_swarm_mode_for_thread_emits_event_with_real_thread_id() {
9638        let runtime = Runtime::fake().unwrap();
9639        let mut events = runtime.subscribe_events();
9640        runtime
9641            .set_agent_swarm_mode_for_thread(
9642                "thread-xyz",
9643                true,
9644                roder_api::subagents::AgentSwarmModeTrigger::Task,
9645            )
9646            .await;
9647        let mut saw = false;
9648        for _ in 0..8 {
9649            let envelope = tokio::time::timeout(std::time::Duration::from_secs(2), events.recv())
9650                .await
9651                .unwrap()
9652                .unwrap();
9653            if let RoderEvent::AgentSwarmModeChanged(event) = envelope.event {
9654                assert_eq!(event.thread_id, "thread-xyz");
9655                assert!(event.enabled);
9656                assert_eq!(
9657                    event.trigger,
9658                    roder_api::subagents::AgentSwarmModeTrigger::Task
9659                );
9660                saw = true;
9661                break;
9662            }
9663        }
9664        assert!(
9665            saw,
9666            "expected an AgentSwarmModeChanged event for the thread"
9667        );
9668    }
9669}