Skip to main content

roder_core/
runtime.rs

1use std::collections::HashMap;
2use std::path::PathBuf;
3use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
4use std::sync::{Arc, Weak};
5
6use anyhow::Context;
7use futures::StreamExt;
8use futures::future::{AbortHandle, Abortable, BoxFuture, try_join_all};
9use roder_api::catalog::{
10    EDIT_TOOL_EDIT, EDIT_TOOL_PATCH, PROVIDER_GEMINI, REASONING_NONE, REASONING_ULTRA,
11    built_in_model_profile, built_in_model_profile_for_provider, lookup_model,
12    model_supports_reasoning_effort,
13};
14use roder_api::context::PolicyGate;
15use roder_api::events::*;
16use roder_api::extension::ExtensionRegistry;
17use roder_api::inference::{
18    AgentInferenceRequest, HostedWebSearchConfig, HostedWebSearchMode, InferenceEngine,
19    InferenceEvent, InferenceTurnContext, InstructionBundle, ModelHarnessProfile,
20    ModelSchemaPolicy, ModelSelection, OutputConfig, ProviderTurnCleanup, ReasoningConfig,
21    RuntimeHints, RuntimeProfile, TokenUsage, ToolCallCompleted, ToolSearchConfig,
22    ToolSearchConfigOverlay, finish_reason_from_stop_reason,
23};
24use roder_api::inference_routing::{InferenceRoutingOutcome, ModelSelectionMode};
25use roder_api::lifecycle::{
26    LifecycleMetricsSnapshot, TurnCleanupOwnership, TurnCleanupState, TurnLifecycleReason,
27    TurnLifecycleRecord, TurnLifecycleSnapshot, TurnLifecycleState, turn_lifecycle_snapshot,
28};
29use roder_api::policy_mode::{PolicyDecision, PolicyMode};
30use roder_api::reliability::{
31    ReliabilityContext, ReliabilityDetails, ReliabilityErrorClass, ReliabilityLimitDecision,
32    ReliabilityLimitRecorded, ReliabilityRequestPolicy, ReliabilityRetryDecision,
33    ReliabilityRetryRecorded, provider_retry_delay_ms,
34};
35use roder_api::remote_runner::{
36    RemoteRunnerProvider, RemoteRunnerSession, RemoteWorkspace, RunnerDestination,
37    RunnerSessionState, ThreadRunnerBinding,
38};
39use roder_api::subagents::SubagentDefinition;
40use roder_api::teams::{
41    TeamId, TeamMailboxMessage, TeamMailboxMessageKind, TeamMemberDescriptor, TeamMemberRole,
42    TeamMemberStatus,
43};
44use roder_api::thread::{
45    ThreadItemEvent, ThreadItemEventKind, ThreadMetadata, ThreadSnapshot, ThreadStore,
46    ThreadUsageMetadata, is_synthetic_event_thread_id, validate_thread_workspace,
47};
48use roder_api::tools::{ToolCall, ToolChoice, ToolExecutionContext, ToolRegistry, ToolResult};
49use roder_api::transcript::{
50    AssistantMessage, ErrorRecord, InputImage, ReasoningSummary, ToolCallRecord, ToolResultRecord,
51    TranscriptItem, UserMessage,
52};
53use roder_sandbox::ScopedFilesystem;
54use roder_sandbox::process::LocalProcessRunner;
55use roder_skills::{SkillRegistry, SkillRegistryOptions};
56use time::{Duration, OffsetDateTime};
57use tokio::sync::{Mutex, Notify, RwLock, oneshot};
58
59mod codex_v2;
60
61use crate::artifacts::{
62    ContextArtifactStore as FilesystemContextArtifactStore, default_context_artifact_dir,
63};
64use crate::bus::EventBus;
65use crate::dynamic_workflows::{
66    DynamicWorkflowEffortProfile, RuntimeDynamicWorkflowConfig, WorkflowTriggerDecision,
67    classify_workflow_trigger, ultracode_reasoning_level_for_model,
68};
69use crate::fake_provider::FakeInferenceEngine;
70use crate::goals::RuntimeGoalController;
71use crate::inference_routing::{
72    InferenceRoutingRequest, RuntimeInferenceRouterConfig, collect_inference_routing_candidates,
73    route_inference_selection, transcript_failure_count_since,
74};
75use crate::instructions::{
76    apply_agent_swarm_mode, apply_codex_multi_agent_mode, apply_model_instruction_overlay,
77    apply_parallel_web_tools, apply_plan_mode, apply_runtime_profile, apply_task_ledger_required,
78    apply_thread_developer_instructions, apply_turn_developer_context,
79};
80use crate::policy_gate::DefaultPolicyGate;
81use crate::reliability::{
82    ReliabilityLimitHit, RuntimeReliabilityConfig, TurnReliabilityState,
83    provider_stream_retry_cause,
84};
85pub use crate::speed_policy::RuntimeSpeedPolicyConfig;
86use crate::speed_policy::{SpeedPolicyState, reasoning_from_decision};
87use crate::subagent_traces::{RuntimeAgentSwarmProgressSink, RuntimeSubagentTraceSink};
88use crate::teams::{TeamManager, TeamMemberStartRequest, TeamStartRequest, TeamState};
89use crate::thread_item_cache::{ThreadItemCache, ThreadItemCacheEntry};
90use crate::verification_gate::VerificationGateState;
91
92const MAX_TOOL_ROUNDS_PER_TURN: usize = 1024;
93/// Injected when `continue_on_failure_limit` turns a consecutive-tool-failure
94/// limit into a continuation, or as the empty-tool-call persistence nudge, so
95/// the model keeps working instead of ending the turn early.
96const RELIABILITY_CONTINUATION_PROMPT: &str = "Verify the task is fully complete. If any part remains unfinished or unverified, keep working using the available tools — try an alternative approach if one is failing. Only stop once the solution is complete and verified; if it is already complete, restate your final answer.";
97/// Capacity of the runtime event broadcast ring buffer. The TUI drains this on
98/// its render loop, which during an active turn only wakes at the ~6 FPS status
99/// animation cadence (backend events do not wake the input poll), so up to
100/// ~166ms of events buffer between drains. A reasoning-high provider that runs
101/// its whole tool loop inside one turn (e.g. claude-code) streams a firehose of
102/// `InferenceEventReceived` deltas plus per-step tool/thinking events; the old
103/// 1024-slot buffer overflowed in those windows and silently dropped tool and
104/// thinking rows from the live view. Sized for generous headroom across bursts
105/// and brief render stalls.
106const EVENT_BUS_CAPACITY: usize = 16_384;
107pub(crate) const FINAL_ANSWER_PHASE: &str = "final_answer";
108pub(crate) const TASK_LEDGER_TOOL_NAME: &str = "task_ledger.update";
109const TASK_LEDGER_COMPLETION_REMINDER_LIMIT: u8 = 2;
110const TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS: u64 = 180;
111const TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT: u8 = 1;
112pub(crate) const MIN_CHILD_DEADLINE_SECONDS: u64 = 2;
113const MODEL_PROFILE_TRACE_KIND: &str = "model_profile_segment";
114const MODEL_SWITCH_SUMMARY_PREFIX: &str = "Model switch summary:";
115const PROVIDER_CLEANUP_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5);
116
117#[derive(Clone, Copy, Debug, Eq, PartialEq)]
118enum InferenceTimeoutAction {
119    ScoreableCheckpoint,
120    Finalization,
121}
122
123#[derive(Debug, Clone)]
124pub struct RuntimeConfig {
125    pub default_provider: String,
126    pub default_model: String,
127    pub reasoning: Option<String>,
128    pub auto_compact_token_limit: Option<u32>,
129    pub file_backed_dynamic_context: bool,
130    pub hosted_web_search: HostedWebSearchConfig,
131    pub tool_search: ToolSearchConfig,
132    pub provider_tool_search: HashMap<String, ToolSearchConfigOverlay>,
133    pub model_tool_search: HashMap<String, ToolSearchConfigOverlay>,
134    pub model_edit_tools: HashMap<String, String>,
135    pub model_parallel_tool_calls: HashMap<String, bool>,
136    pub model_profiles: HashMap<String, ModelHarnessProfile>,
137    pub tool_allowlist: Vec<String>,
138    /// Seconds a host-executed external tool call may stay unresolved before it fails with a timeout error.
139    pub external_tool_timeout_seconds: u64,
140    pub command_shell: String,
141    pub workspace: Option<String>,
142    pub policy_mode: PolicyMode,
143    /// Whether agent-swarm mode is active (roadmap 104). When on, the runtime
144    /// injects the swarm reminder into each turn's developer instructions so any
145    /// client benefits, not just the TUI.
146    pub agent_swarm_mode: bool,
147    /// Whether ultra mode is active. When on, the runtime injects proactive
148    /// multi-agent delegation policy for any model (not only Codex Sol/Terra
149    /// Ultra effort). Selecting Ultra reasoning on Sol/Terra still enables
150    /// proactive multi-agent for that model without requiring this flag.
151    pub ultra_mode: bool,
152    pub runtime_profile: RuntimeProfile,
153    pub inference_router: RuntimeInferenceRouterConfig,
154    pub speed_policy: RuntimeSpeedPolicyConfig,
155    pub dynamic_workflows: RuntimeDynamicWorkflowConfig,
156    pub reliability: RuntimeReliabilityConfig,
157    /// Positive wall-clock limit for a turn in any runtime profile. `None` or zero is unbounded.
158    pub turn_deadline_seconds: Option<u64>,
159    pub remote_runner_destination: Option<RunnerDestination>,
160    pub team_data_dir: Option<PathBuf>,
161    pub roadmap_data_dir: Option<PathBuf>,
162    pub media_generation: crate::media_generation::RuntimeMediaGenerationConfig,
163    /// `[review]` settings: the review sub-turn and its publishers.
164    pub review: crate::review::RuntimeReviewConfig,
165}
166
167impl Default for RuntimeConfig {
168    fn default() -> Self {
169        Self {
170            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
171            default_model: "mock".to_string(),
172            reasoning: None,
173            auto_compact_token_limit: None,
174            file_backed_dynamic_context: true,
175            hosted_web_search: HostedWebSearchConfig::cached(),
176            tool_search: ToolSearchConfig::default(),
177            provider_tool_search: HashMap::new(),
178            model_tool_search: HashMap::new(),
179            model_edit_tools: HashMap::new(),
180            model_parallel_tool_calls: HashMap::new(),
181            model_profiles: HashMap::new(),
182            tool_allowlist: Vec::new(),
183            external_tool_timeout_seconds: DEFAULT_EXTERNAL_TOOL_TIMEOUT_SECONDS,
184            command_shell: roder_api::command_shell::default_command_shell(),
185            workspace: None,
186            policy_mode: PolicyMode::Default,
187            agent_swarm_mode: false,
188            ultra_mode: false,
189            runtime_profile: RuntimeProfile::Interactive,
190            inference_router: RuntimeInferenceRouterConfig::default(),
191            speed_policy: RuntimeSpeedPolicyConfig::default(),
192            dynamic_workflows: RuntimeDynamicWorkflowConfig::default(),
193            reliability: RuntimeReliabilityConfig::default(),
194            turn_deadline_seconds: None,
195            remote_runner_destination: None,
196            team_data_dir: None,
197            roadmap_data_dir: None,
198            media_generation: crate::media_generation::RuntimeMediaGenerationConfig::default(),
199            review: crate::review::RuntimeReviewConfig::default(),
200        }
201    }
202}
203
204#[derive(Debug, Clone)]
205pub struct StartTurnRequest {
206    pub thread_id: ThreadId,
207    pub message: String,
208    pub images: Vec<InputImage>,
209    pub provider_override: Option<String>,
210    pub model_override: Option<String>,
211    pub reasoning_override: Option<String>,
212    pub workspace: String,
213    pub instructions: InstructionBundle,
214    /**
215     * Per-turn developer-authority context for this turn's InstructionBundle.
216     * Applies to every inference round of the turn, is never written to
217     * thread state, and does not carry over to later turns.
218     */
219    pub developer_context: Option<String>,
220    pub task_ledger_required: bool,
221}
222
223#[derive(Debug, Clone)]
224pub struct CreateThreadRequest {
225    pub title: Option<String>,
226    pub workspace: String,
227    pub workspace_id: Option<String>,
228    pub root_id: Option<String>,
229    pub provider: Option<String>,
230    pub model: Option<String>,
231    pub selection_mode: Option<ModelSelectionMode>,
232    /// Per-thread tool filter applied on top of the runtime allowlist. Empty = no filtering.
233    pub tool_allowlist: Vec<String>,
234    /// Host-supplied instructions added to the developer slot of every turn's inference request.
235    pub developer_instructions: Option<String>,
236    /// Host-executed tool specs advertised to the model on every turn of this thread.
237    pub external_tools: Vec<roder_api::tools::ToolSpec>,
238    /// Explicit remote-runner binding for the thread's native coding tools.
239    pub runner: Option<ThreadRunnerSelection>,
240}
241
242/**
243 * Thread-level remote-runner selection. The destination config is persisted
244 * with the thread, so secrets must reach the provider through its
245 * environment, not this config.
246 */
247#[derive(Debug, Clone)]
248pub struct ThreadRunnerSelection {
249    pub provider_id: String,
250    pub config: serde_json::Value,
251    /// Absolute path on the runner used as the thread's coding-tool workspace root.
252    pub workspace: String,
253    /**
254     * Extra absolute runner paths file reads may resolve under, beyond
255     * `workspace`. Writes and the working directory stay confined to
256     * `workspace`.
257     */
258    pub read_roots: Vec<String>,
259}
260
261#[derive(Debug, Clone, PartialEq, Eq)]
262pub struct PendingPlanExit {
263    pub thread_id: ThreadId,
264    pub turn_id: TurnId,
265    pub request_id: String,
266    pub target_mode: PolicyMode,
267    pub plan_summary: Option<String>,
268    pub next_steps: Vec<String>,
269    pub requested_at: OffsetDateTime,
270    pub expires_at: Option<OffsetDateTime>,
271}
272
273pub(crate) struct PendingToolApproval {
274    pub(crate) thread_id: ThreadId,
275    pub(crate) turn_id: TurnId,
276    pub(crate) tool_id: String,
277    pub(crate) tool_name: String,
278    pub(crate) call: roder_api::tools::ToolCall,
279    pub(crate) tx: oneshot::Sender<bool>,
280}
281
282pub(crate) struct PendingUserInput {
283    pub(crate) thread_id: ThreadId,
284    pub(crate) turn_id: TurnId,
285    pub(crate) tx: oneshot::Sender<serde_json::Value>,
286}
287
288/// Host answer to an external tool call delivered via `tools/resolve`.
289#[derive(Debug, Clone, PartialEq, Eq)]
290pub struct ExternalToolResolution {
291    pub output: String,
292    pub is_error: bool,
293}
294
295pub(crate) struct PendingExternalToolCall {
296    pub(crate) thread_id: ThreadId,
297    pub(crate) turn_id: TurnId,
298    pub(crate) tool_id: String,
299    pub(crate) tool_name: String,
300    pub(crate) tx: oneshot::Sender<ExternalToolResolution>,
301}
302
303#[derive(Clone)]
304struct ActiveTurnHandle {
305    thread_id: ThreadId,
306    abort: AbortHandle,
307    steers: Arc<Mutex<Vec<QueuedTurnSteer>>>,
308    drain: Arc<TurnDrainHandle>,
309}
310
311/// Tracks a task after it has been interrupted and removed from the interactive
312/// active-turn map. This lets users start a follow-up turn immediately while a
313/// bounded runtime shutdown can still await the original task's cleanup.
314struct TurnDrainHandle {
315    thread_id: ThreadId,
316    interrupt_requested: AtomicBool,
317    interrupt_reason: Mutex<Option<TurnLifecycleReason>>,
318    completed: AtomicBool,
319    completed_notify: Notify,
320}
321
322/// Result of a bounded runtime drain. `Clean` proves the runtime's own turn
323/// tasks have reached their cleanup paths; provider-specific child-process
324/// reaping remains an explicit provider capability.
325#[derive(Debug, Clone, PartialEq, Eq)]
326pub enum RuntimeDrainOutcome {
327    Clean {
328        interrupted_turn_ids: Vec<TurnId>,
329    },
330    DeadlineExceeded {
331        interrupted_turn_ids: Vec<TurnId>,
332        remaining_turn_ids: Vec<TurnId>,
333    },
334    /// At least one lifecycle transition initiated by this drain could not be
335    /// durably appended. The runtime still made its best cleanup attempt, but
336    /// callers must not claim a persisted terminal state as proof of recovery.
337    PersistenceFailed {
338        interrupted_turn_ids: Vec<TurnId>,
339        remaining_turn_ids: Vec<TurnId>,
340    },
341}
342
343#[derive(Clone)]
344struct QueuedTurnSteer {
345    message: UserMessage,
346    mailbox_ack: Option<MailboxDeliveryAck>,
347}
348
349#[derive(Clone)]
350struct MailboxDeliveryAck {
351    team_id: TeamId,
352    message_ids: Vec<String>,
353}
354
355#[derive(Clone)]
356struct InheritedTurnContext {
357    workspace: String,
358    instructions: InstructionBundle,
359    developer_context: Option<String>,
360}
361
362#[derive(Debug, Clone, Default, PartialEq, Eq)]
363pub struct ThreadActivity {
364    pub active_turn_id: Option<TurnId>,
365    pub active_flags: Vec<String>,
366}
367
368/// Per-thread settings persisted at thread creation and applied to every turn.
369#[derive(Debug, Clone, Default)]
370pub(crate) struct ThreadTurnOverrides {
371    pub(crate) tool_allowlist: Vec<String>,
372    pub(crate) developer_instructions: Option<String>,
373    pub(crate) external_tools: Vec<roder_api::tools::ToolSpec>,
374}
375
376#[derive(Debug, Clone, Copy, PartialEq, Eq)]
377pub(crate) enum TurnRunOutcome {
378    Completed,
379    Stopped,
380}
381
382impl PendingPlanExit {
383    pub fn new(
384        thread_id: ThreadId,
385        turn_id: TurnId,
386        request_id: String,
387        target_mode: PolicyMode,
388        plan_summary: Option<String>,
389        next_steps: Vec<String>,
390    ) -> Self {
391        let requested_at = OffsetDateTime::now_utc();
392        Self {
393            thread_id,
394            turn_id,
395            request_id,
396            target_mode,
397            plan_summary,
398            next_steps,
399            requested_at,
400            expires_at: Some(requested_at + default_plan_exit_timeout()),
401        }
402    }
403
404    pub fn is_expired(&self, now: OffsetDateTime) -> bool {
405        self.expires_at.is_some_and(|expires_at| now >= expires_at)
406    }
407}
408
409pub fn default_plan_exit_timeout() -> Duration {
410    Duration::minutes(10)
411}
412
413pub const DEFAULT_EXTERNAL_TOOL_TIMEOUT_SECONDS: u64 = 300;
414
415fn format_mailbox_messages(team: &TeamState, messages: &[TeamMailboxMessage]) -> String {
416    messages
417        .iter()
418        .map(|message| {
419            let recipient = team
420                .members
421                .iter()
422                .find(|member| member.id == message.to_member_id)
423                .map(canonical_team_member_path)
424                .unwrap_or_else(|| format!("/root/{}", message.to_member_id));
425            let sender = message
426                .from_member_id
427                .as_deref()
428                .and_then(|id| team.members.iter().find(|member| member.id == id))
429                .map(canonical_team_member_path)
430                .unwrap_or_else(|| "/root".to_string());
431            let message_type = match message.kind {
432                TeamMailboxMessageKind::Message => "MESSAGE",
433                TeamMailboxMessageKind::NewTask => "NEW_TASK",
434                TeamMailboxMessageKind::FinalAnswer => "FINAL_ANSWER",
435            };
436            format!(
437                "Message Type: {message_type}\nTask name: {recipient}\nSender: {sender}\nPayload:\n{}",
438                message.text
439            )
440        })
441        .collect::<Vec<_>>()
442        .join("\n\n")
443}
444
445fn canonical_team_member_path(member: &TeamMemberDescriptor) -> String {
446    if let Some(agent_path) = member
447        .agent_path
448        .as_deref()
449        .filter(|path| *path == "/root" || path.starts_with("/root/"))
450    {
451        return agent_path.to_string();
452    }
453    if member.role == TeamMemberRole::Lead {
454        return "/root".to_string();
455    }
456    member
457        .task_name
458        .as_deref()
459        .filter(|name| !name.trim().is_empty())
460        .map(|name| format!("/root/{}", name.trim().trim_matches('/')))
461        .unwrap_or_else(|| format!("/root/{}", member.id))
462}
463
464fn is_codex_v2_team(team: &TeamState) -> bool {
465    team.members.iter().any(|member| {
466        member
467            .model
468            .as_deref()
469            .is_some_and(|model| model_supports_reasoning_effort(model, REASONING_ULTRA))
470    })
471}
472
473fn runtime_local_team_view(
474    mut team: TeamState,
475    active_thread_ids: &std::collections::HashSet<ThreadId>,
476) -> TeamState {
477    for member in &mut team.members {
478        if member.status == TeamMemberStatus::Running
479            && !active_thread_ids.contains(&member.thread_id)
480        {
481            member.status = TeamMemberStatus::Interrupted;
482            member.current_turn_id = None;
483        }
484    }
485    team
486}
487
488type RunnerToolExecutionKey = (String, String);
489type RunnerToolExecutionMutex = Mutex<()>;
490type RunnerToolExecutionLockRegistry =
491    Mutex<HashMap<RunnerToolExecutionKey, Weak<RunnerToolExecutionMutex>>>;
492
493pub struct Runtime {
494    pub bus: EventBus,
495    pub registry: ExtensionRegistry,
496    config: RwLock<RuntimeConfig>,
497    pending_plan_exit: RwLock<Option<PendingPlanExit>>,
498    pub(crate) pending_tool_approvals: Mutex<HashMap<String, PendingToolApproval>>,
499    pub(crate) pending_user_inputs: Mutex<HashMap<String, PendingUserInput>>,
500    pub(crate) pending_external_tool_calls: Mutex<HashMap<String, PendingExternalToolCall>>,
501    /// Serializes turn admission with shutdown quiescing. A drain takes this
502    /// gate before flipping `accepting_turns` and snapshotting active work so a
503    /// turn that observed the old flag cannot be inserted after the snapshot.
504    turn_admission: Mutex<()>,
505    active_turns: RwLock<HashMap<TurnId, ActiveTurnHandle>>,
506    turn_drains: RwLock<HashMap<TurnId, Arc<TurnDrainHandle>>>,
507    /// Provider cleanup handles register synchronously through the inference
508    /// tool-executor context. They are intentionally separate from active turn
509    /// admission so an interrupted turn can remain drainable after a follow-up
510    /// turn starts on the same thread.
511    provider_turn_cleanups: std::sync::Mutex<HashMap<TurnId, Arc<dyn ProviderTurnCleanup>>>,
512    accepting_turns: AtomicBool,
513    /// Monotonic counter used by bounded drains to surface lifecycle-state
514    /// persistence failures without making an individual provider turn fail.
515    lifecycle_persistence_failures: AtomicUsize,
516    lifecycle_shutdown_drains: AtomicUsize,
517    lifecycle_clean_shutdowns: AtomicUsize,
518    lifecycle_deadline_exceeded: AtomicUsize,
519    lifecycle_persistence_failed_drains: AtomicUsize,
520    lifecycle_restart_reconciliations: AtomicUsize,
521    lifecycle_shutdown_drain_duration_ms_total: AtomicUsize,
522    provider_cleanup_confirmed: AtomicUsize,
523    provider_cleanup_timed_out: AtomicUsize,
524    provider_cleanup_unknown: AtomicUsize,
525    active_turns_changed: Notify,
526    active_turn_selections: RwLock<HashMap<TurnId, ModelSelectionMode>>,
527    active_turn_contexts: RwLock<HashMap<TurnId, InheritedTurnContext>>,
528    /// Process-local execution boundary. Local/CLI runtimes default to true;
529    /// hosted runtime pools set this false so missing runner metadata fails
530    /// closed instead of exposing the host workspace.
531    allow_local_workspaces: AtomicBool,
532    // Spawn-time live authority retained for every reusable turn of a long-lived teammate.
533    // This stays process-local because developer_context is explicitly volatile and must never
534    // be persisted to thread state.
535    team_member_turn_contexts: Mutex<HashMap<ThreadId, InheritedTurnContext>>,
536    workspace: PathBuf,
537    teams: TeamManager,
538    agent_team_spawn_lock: Mutex<()>,
539    pub(crate) roadmaps: Mutex<roder_roadmap::RoadmapRuntime>,
540    /// Completed reviews, most recent last, so a later publish can resolve a
541    /// review id back to its findings. Bounded; see `review::run`.
542    pub(crate) reviews: Mutex<Vec<crate::review::ReviewRecord>>,
543    pub(crate) goals: Arc<RuntimeGoalController>,
544    context_artifacts: roder_api::artifacts::ContextArtifactStore,
545    pub(crate) thread_store: Option<Arc<dyn ThreadStore>>,
546    thread_item_cache: Mutex<ThreadItemCache>,
547    pub(crate) tool_registry: ToolRegistry,
548    media_generation: Arc<crate::media_generation::MediaGenerationService>,
549    pub(crate) skills: RwLock<SkillRegistry>,
550    /// Lazily-started bounded dispatch of emitted events to registry
551    /// `EventSink`s (process extensions etc.); see `event_sink_dispatch`.
552    event_sink_dispatcher: tokio::sync::OnceCell<crate::event_sink_dispatch::EventSinkDispatcher>,
553    pub(crate) compaction_hysteresis: std::sync::Mutex<HashMap<ThreadId, u32>>,
554    /// Per-thread agent-swarm mode overrides (roadmap 104). A thread present in
555    /// this map uses its stored value; absent threads fall back to the
556    /// runtime-global `RuntimeConfig.agent_swarm_mode` default. This mirrors the
557    /// team per-member policy-mode override idiom so swarm mode is per-thread
558    /// like the other `thread/*` operations, without a separate runtime.
559    agent_swarm_modes: RwLock<HashMap<ThreadId, bool>>,
560    /// Per-thread ultra mode overrides. A thread present in this map uses its
561    /// stored value; absent threads fall back to the runtime-global
562    /// `RuntimeConfig.ultra_mode` default.
563    ultra_modes: RwLock<HashMap<ThreadId, bool>>,
564    /**
565     * Live remote-runner sessions keyed by thread. Each per-thread mutex is
566     * held across provider initialization, making the first workspace-tool
567     * access a singleflight while allowing unrelated threads to initialize in
568     * parallel. Failed initialization is shared with current waiters, then
569     * evicted so a later tool call can retry.
570     */
571    runner_sessions: Arc<Mutex<HashMap<ThreadId, Arc<RunnerSessionSlot>>>>,
572    /**
573     * Outer tool-call locks keyed by the actual remote session, not the
574     * per-thread destination id. Hosted runtimes are tenant-scoped, so this
575     * serializes threads that share one sandbox without coupling different
576     * tenants or independent runner sessions.
577     */
578    runner_tool_execution_locks: RunnerToolExecutionLockRegistry,
579}
580
581#[derive(Clone)]
582struct CachedRunnerSession {
583    destination: RunnerDestination,
584    session: Arc<dyn RemoteRunnerSession>,
585}
586
587struct RunnerSessionSlot {
588    provider_id: String,
589    destination_id: String,
590    state: Mutex<RunnerSessionSlotState>,
591    initialized: Notify,
592}
593
594enum RunnerSessionSlotState {
595    Empty,
596    Initializing,
597    Ready(CachedRunnerSession),
598    Failed(Arc<str>),
599}
600
601impl RunnerSessionSlot {
602    fn empty(destination: &RunnerDestination) -> Self {
603        Self {
604            provider_id: destination.provider_id.clone(),
605            destination_id: destination.id.clone(),
606            state: Mutex::new(RunnerSessionSlotState::Empty),
607            initialized: Notify::new(),
608        }
609    }
610
611    fn ready(session: CachedRunnerSession) -> Self {
612        Self {
613            provider_id: session.destination.provider_id.clone(),
614            destination_id: session.destination.id.clone(),
615            state: Mutex::new(RunnerSessionSlotState::Ready(session)),
616            initialized: Notify::new(),
617        }
618    }
619
620    fn matches(&self, destination: &RunnerDestination) -> bool {
621        self.provider_id == destination.provider_id && self.destination_id == destination.id
622    }
623}
624
625impl Runtime {
626    pub fn new(registry: ExtensionRegistry, config: RuntimeConfig) -> anyhow::Result<Self> {
627        if registry.inference_engines.is_empty() {
628            anyhow::bail!("at least one inference engine must be registered");
629        }
630        validate_runtime_config_reasoning(&config)?;
631        validate_runtime_inference_router_config(&registry, &config)?;
632
633        let bus = EventBus::new(EVENT_BUS_CAPACITY);
634        let thread_store = registry
635            .thread_stores
636            .first()
637            .map(|factory| factory.create());
638        let mut tool_registry = ToolRegistry::default();
639        for contributor in &registry.tools {
640            contributor
641                .contribute(&mut tool_registry)
642                .with_context(|| format!("tool contributor {} failed", contributor.id()))?;
643        }
644        crate::agent_control_tools::contribute_agent_control_tools(&mut tool_registry)?;
645
646        let media_generation = Arc::new(crate::media_generation::MediaGenerationService::new(
647            registry.media_generator_providers.clone(),
648            config.media_generation.clone(),
649        ));
650        tool_registry.replace(Arc::new(
651            crate::media_generation::MediaGenerateImageTool::new(media_generation.clone()),
652        ));
653
654        let team_data_dir = config.team_data_dir.clone();
655        let workspace = config
656            .workspace
657            .clone()
658            .map(PathBuf::from)
659            .unwrap_or(std::env::current_dir()?);
660        let roadmap_data_dir = config
661            .roadmap_data_dir
662            .clone()
663            .unwrap_or_else(|| workspace.join(".roder"));
664        let context_artifacts = thread_store
665            .as_ref()
666            .and_then(|store| store.context_artifact_store())
667            .or_else(|| {
668                thread_store
669                    .as_ref()
670                    .and_then(|store| store.local_thread_root())
671                    .map(FilesystemContextArtifactStore::shared_thread_scoped)
672            })
673            .unwrap_or_else(|| {
674                FilesystemContextArtifactStore::shared_legacy(default_context_artifact_dir())
675            });
676        let goals = Arc::new(RuntimeGoalController::new(
677            bus.clone(),
678            thread_store.clone(),
679        ));
680        let runtime = Self {
681            bus,
682            registry,
683            config: RwLock::new(config),
684            pending_plan_exit: RwLock::new(None),
685            pending_tool_approvals: Mutex::new(HashMap::new()),
686            pending_user_inputs: Mutex::new(HashMap::new()),
687            pending_external_tool_calls: Mutex::new(HashMap::new()),
688            turn_admission: Mutex::new(()),
689            active_turns: RwLock::new(HashMap::new()),
690            turn_drains: RwLock::new(HashMap::new()),
691            provider_turn_cleanups: std::sync::Mutex::new(HashMap::new()),
692            accepting_turns: AtomicBool::new(true),
693            lifecycle_persistence_failures: AtomicUsize::new(0),
694            lifecycle_shutdown_drains: AtomicUsize::new(0),
695            lifecycle_clean_shutdowns: AtomicUsize::new(0),
696            lifecycle_deadline_exceeded: AtomicUsize::new(0),
697            lifecycle_persistence_failed_drains: AtomicUsize::new(0),
698            lifecycle_restart_reconciliations: AtomicUsize::new(0),
699            lifecycle_shutdown_drain_duration_ms_total: AtomicUsize::new(0),
700            provider_cleanup_confirmed: AtomicUsize::new(0),
701            provider_cleanup_timed_out: AtomicUsize::new(0),
702            provider_cleanup_unknown: AtomicUsize::new(0),
703            active_turns_changed: Notify::new(),
704            active_turn_selections: RwLock::new(HashMap::new()),
705            active_turn_contexts: RwLock::new(HashMap::new()),
706            allow_local_workspaces: AtomicBool::new(true),
707            team_member_turn_contexts: Mutex::new(HashMap::new()),
708            workspace: workspace.clone(),
709            teams: TeamManager::new(
710                team_data_dir.unwrap_or_else(crate::teams::default_team_data_dir),
711            ),
712            agent_team_spawn_lock: Mutex::new(()),
713            roadmaps: Mutex::new(roder_roadmap::RoadmapRuntime::new(
714                workspace,
715                roadmap_data_dir,
716            )),
717            reviews: Mutex::new(Vec::new()),
718            goals,
719            context_artifacts,
720            thread_store,
721            thread_item_cache: Mutex::new(ThreadItemCache::default()),
722            tool_registry,
723            media_generation,
724            skills: RwLock::new(SkillRegistry::load(SkillRegistryOptions::new(
725                PathBuf::new(),
726            ))),
727            event_sink_dispatcher: tokio::sync::OnceCell::new(),
728            compaction_hysteresis: crate::compaction_runtime::compaction_hysteresis_state(),
729            agent_swarm_modes: RwLock::new(HashMap::new()),
730            ultra_modes: RwLock::new(HashMap::new()),
731            runner_sessions: Arc::new(Mutex::new(HashMap::new())),
732            runner_tool_execution_locks: Mutex::new(HashMap::new()),
733        };
734        runtime.bus.emit(RoderEvent::RuntimeStarted(RuntimeStarted {
735            timestamp: OffsetDateTime::now_utc(),
736        }));
737        for manifest in &runtime.registry.manifests {
738            runtime
739                .bus
740                .emit(RoderEvent::ExtensionRegistered(ExtensionRegistered {
741                    extension_id: manifest.id.clone(),
742                    timestamp: OffsetDateTime::now_utc(),
743                }));
744        }
745        Ok(runtime)
746    }
747
748    pub fn from_engine(engine: Arc<dyn InferenceEngine>) -> anyhow::Result<Self> {
749        let mut builder = roder_api::extension::ExtensionRegistryBuilder::new();
750        builder.inference_engine(engine);
751        Self::new(builder.build()?, RuntimeConfig::default())
752    }
753
754    pub fn fake() -> anyhow::Result<Self> {
755        Self::from_engine(Arc::new(FakeInferenceEngine))
756    }
757
758    pub fn subscribe_events(&self) -> tokio::sync::broadcast::Receiver<EventEnvelope> {
759        self.bus.subscribe()
760    }
761
762    /// Returns process-local, redacted lifecycle health counters. The snapshot
763    /// intentionally has fixed fields and contains no provider, command,
764    /// process, thread, or turn identifiers.
765    pub fn lifecycle_metrics(&self) -> LifecycleMetricsSnapshot {
766        LifecycleMetricsSnapshot {
767            shutdown_drain_count: self.lifecycle_shutdown_drains.load(Ordering::Acquire) as u64,
768            clean_shutdown_count: self.lifecycle_clean_shutdowns.load(Ordering::Acquire) as u64,
769            deadline_exceeded_count: self.lifecycle_deadline_exceeded.load(Ordering::Acquire)
770                as u64,
771            persistence_failed_count: self
772                .lifecycle_persistence_failed_drains
773                .load(Ordering::Acquire) as u64,
774            restart_reconciliation_count: self
775                .lifecycle_restart_reconciliations
776                .load(Ordering::Acquire) as u64,
777            lifecycle_persistence_failure_count: self
778                .lifecycle_persistence_failures
779                .load(Ordering::Acquire) as u64,
780            shutdown_drain_duration_ms_total: self
781                .lifecycle_shutdown_drain_duration_ms_total
782                .load(Ordering::Acquire) as u64,
783            provider_cleanup_confirmed_count: self
784                .provider_cleanup_confirmed
785                .load(Ordering::Acquire) as u64,
786            provider_cleanup_timed_out_count: self
787                .provider_cleanup_timed_out
788                .load(Ordering::Acquire) as u64,
789            provider_cleanup_unknown_count: self.provider_cleanup_unknown.load(Ordering::Acquire)
790                as u64,
791        }
792    }
793
794    async fn persist_turn_lifecycle_record(&self, record: &TurnLifecycleRecord) -> bool {
795        let Some(store) = &self.thread_store else {
796            return true;
797        };
798        let state = match record.extension_state() {
799            Ok(state) => state,
800            Err(_) => {
801                self.lifecycle_persistence_failures
802                    .fetch_add(1, Ordering::AcqRel);
803                return false;
804            }
805        };
806        // Lifecycle diagnostics must not turn a provider result into a failed
807        // turn merely because a third-party store lacks extension-state support.
808        // JSONL/Postgres stores persist this append-only record atomically at
809        // their own storage boundary.
810        if store
811            .append_extension_state(&record.thread_id, &state)
812            .await
813            .is_err()
814        {
815            self.lifecycle_persistence_failures
816                .fetch_add(1, Ordering::AcqRel);
817            return false;
818        }
819        true
820    }
821
822    async fn record_turn_lifecycle(
823        &self,
824        thread_id: ThreadId,
825        turn_id: TurnId,
826        state: TurnLifecycleState,
827        cleanup: TurnCleanupState,
828        reason: Option<TurnLifecycleReason>,
829    ) -> TurnLifecycleRecord {
830        self.record_turn_lifecycle_with_ownership(
831            thread_id,
832            turn_id,
833            state,
834            cleanup,
835            reason,
836            TurnCleanupOwnership::RuntimeTaskOnly,
837        )
838        .await
839    }
840
841    async fn record_turn_lifecycle_with_ownership(
842        &self,
843        thread_id: ThreadId,
844        turn_id: TurnId,
845        state: TurnLifecycleState,
846        cleanup: TurnCleanupState,
847        reason: Option<TurnLifecycleReason>,
848        ownership: TurnCleanupOwnership,
849    ) -> TurnLifecycleRecord {
850        let record = TurnLifecycleRecord::new(
851            thread_id,
852            turn_id,
853            state,
854            cleanup,
855            reason,
856            OffsetDateTime::now_utc(),
857        )
858        .with_ownership(ownership);
859        let _ = self.persist_turn_lifecycle_record(&record).await;
860        self.emit(RoderEvent::TurnLifecycleUpdated(record.clone()))
861            .await;
862        record
863    }
864
865    pub(crate) fn register_provider_turn_cleanup(
866        &self,
867        turn_id: &TurnId,
868        cleanup: Arc<dyn ProviderTurnCleanup>,
869    ) {
870        if let Ok(mut cleanups) = self.provider_turn_cleanups.lock() {
871            cleanups.insert(turn_id.clone(), cleanup);
872        }
873    }
874
875    fn provider_cleanup_ownership(&self, turn_id: &TurnId) -> TurnCleanupOwnership {
876        self.provider_turn_cleanups
877            .lock()
878            .ok()
879            .and_then(|cleanups| cleanups.get(turn_id).cloned())
880            .map(|cleanup| cleanup.ownership())
881            .unwrap_or(TurnCleanupOwnership::RuntimeTaskOnly)
882    }
883
884    async fn await_provider_turn_cleanup(
885        &self,
886        turn_id: &TurnId,
887    ) -> (TurnCleanupState, TurnCleanupOwnership) {
888        let cleanup = self
889            .provider_turn_cleanups
890            .lock()
891            .ok()
892            .and_then(|mut cleanups| cleanups.remove(turn_id));
893        let Some(cleanup) = cleanup else {
894            return (
895                TurnCleanupState::Unknown,
896                TurnCleanupOwnership::RuntimeTaskOnly,
897            );
898        };
899        let ownership = cleanup.ownership();
900        match tokio::time::timeout(PROVIDER_CLEANUP_TIMEOUT, cleanup.wait_for_cleanup()).await {
901            Ok(Ok(())) => {
902                self.provider_cleanup_confirmed
903                    .fetch_add(1, Ordering::AcqRel);
904                (
905                    TurnCleanupState::Completed,
906                    TurnCleanupOwnership::ProviderCleanupConfirmed,
907                )
908            }
909            Ok(Err(_)) => {
910                self.provider_cleanup_unknown.fetch_add(1, Ordering::AcqRel);
911                (TurnCleanupState::Unknown, ownership)
912            }
913            Err(_) => {
914                self.provider_cleanup_timed_out
915                    .fetch_add(1, Ordering::AcqRel);
916                (TurnCleanupState::TimedOut, ownership)
917            }
918        }
919    }
920
921    fn record_lifecycle_drain_outcome(
922        &self,
923        started_at: tokio::time::Instant,
924        outcome: &RuntimeDrainOutcome,
925    ) {
926        self.lifecycle_shutdown_drains
927            .fetch_add(1, Ordering::AcqRel);
928        self.lifecycle_shutdown_drain_duration_ms_total.fetch_add(
929            started_at.elapsed().as_millis().min(usize::MAX as u128) as usize,
930            Ordering::AcqRel,
931        );
932        match outcome {
933            RuntimeDrainOutcome::Clean { .. } => {
934                self.lifecycle_clean_shutdowns
935                    .fetch_add(1, Ordering::AcqRel);
936            }
937            RuntimeDrainOutcome::DeadlineExceeded { .. } => {
938                self.lifecycle_deadline_exceeded
939                    .fetch_add(1, Ordering::AcqRel);
940            }
941            RuntimeDrainOutcome::PersistenceFailed { .. } => {
942                self.lifecycle_persistence_failed_drains
943                    .fetch_add(1, Ordering::AcqRel);
944            }
945        }
946    }
947
948    fn lifecycle_record_for_event(event: &RoderEvent) -> Option<TurnLifecycleRecord> {
949        match event {
950            RoderEvent::TurnStarted(event) => Some(TurnLifecycleRecord::new(
951                event.thread_id.clone(),
952                event.turn_id.clone(),
953                TurnLifecycleState::Running,
954                TurnCleanupState::NotRequested,
955                None,
956                event.timestamp,
957            )),
958            // `run_turn` persists completed turns after it has awaited a
959            // registered provider cleanup handle. Do not let the generic event
960            // projection overwrite that acknowledgement with an unproven state.
961            RoderEvent::TurnCompleted(_) => None,
962            // Some failure paths already have an event before the turn wrapper
963            // reaches its cleanup acknowledgement. Preserve a durable fallback
964            // reason here; the wrapper appends a later record with the more
965            // precise cleanup outcome when a provider registered one.
966            RoderEvent::TurnFailed(event) => Some(TurnLifecycleRecord::new(
967                event.thread_id.clone(),
968                event.turn_id.clone(),
969                TurnLifecycleState::Failed,
970                TurnCleanupState::Unknown,
971                Some(if event.error_kind.as_deref() == Some("deadline_timeout") {
972                    TurnLifecycleReason::DeadlineExceeded
973                } else {
974                    TurnLifecycleReason::ProviderFailure
975                }),
976                event.timestamp,
977            )),
978            RoderEvent::TurnDeadlineExceeded(_) | RoderEvent::TurnInterrupted(_) => None,
979            RoderEvent::TurnLifecycleUpdated(_) => None,
980            _ => None,
981        }
982    }
983
984    pub fn registry(&self) -> &ExtensionRegistry {
985        &self.registry
986    }
987
988    pub fn media_generation(&self) -> Arc<crate::media_generation::MediaGenerationService> {
989        self.media_generation.clone()
990    }
991
992    pub fn context_artifacts(&self) -> roder_api::artifacts::ContextArtifactStore {
993        self.context_artifacts.clone()
994    }
995
996    pub async fn execute_workflow_tool(
997        &self,
998        thread_id: ThreadId,
999        tool_name: &str,
1000        arguments: serde_json::Value,
1001    ) -> anyhow::Result<ToolResult> {
1002        let Some(executor) = self.tool_registry.get(tool_name) else {
1003            anyhow::bail!("tool not found: {tool_name}");
1004        };
1005        let tool_call = ToolCall {
1006            id: format!("slash-{tool_name}"),
1007            name: tool_name.to_string(),
1008            raw_arguments: serde_json::to_string(&arguments)?,
1009            arguments,
1010            thread_id: thread_id.clone(),
1011            turn_id: "slash-command".to_string(),
1012        };
1013        let runtime_config = self.status().await;
1014        let ctx = self.tool_execution_context(
1015            thread_id,
1016            "slash-command".to_string(),
1017            runtime_config.policy_mode,
1018            runtime_config.workspace.as_deref(),
1019            Some(&runtime_config.command_shell),
1020        );
1021        executor.execute(ctx, tool_call).await
1022    }
1023
1024    pub(crate) fn tool_execution_context(
1025        &self,
1026        thread_id: ThreadId,
1027        turn_id: TurnId,
1028        mode: PolicyMode,
1029        workspace: Option<&str>,
1030        command_shell: Option<&str>,
1031    ) -> ToolExecutionContext {
1032        let mut ctx = ToolExecutionContext::new(thread_id, turn_id, mode)
1033            .with_command_shell(command_shell.unwrap_or_default())
1034            .with_process_runner(Arc::new(LocalProcessRunner))
1035            .with_context_artifacts(self.context_artifacts.backend())
1036            .with_goal_controller(self.goals.clone())
1037            .with_subagent_trace_sink(Arc::new(RuntimeSubagentTraceSink::new(
1038                self.bus.clone(),
1039                self.thread_store.clone(),
1040            )))
1041            .with_swarm_progress_sink(Arc::new(RuntimeAgentSwarmProgressSink::new(
1042                self.bus.clone(),
1043                self.thread_store.clone(),
1044            )));
1045        if let Some(workspace) = workspace {
1046            ctx = ctx.with_workspace_handle(Arc::new(ScopedFilesystem::new(workspace)));
1047        }
1048        ctx
1049    }
1050
1051    pub async fn status(&self) -> RuntimeConfig {
1052        self.config.read().await.clone()
1053    }
1054
1055    pub async fn set_skills(&self, skills: SkillRegistry) {
1056        *self.skills.write().await = skills;
1057    }
1058
1059    pub async fn skills_snapshot(&self) -> SkillRegistry {
1060        self.skills.read().await.clone()
1061    }
1062
1063    pub fn workspace(&self) -> PathBuf {
1064        self.workspace.clone()
1065    }
1066
1067    /// Controls whether native workspace tools may execute without an
1068    /// explicit remote-runner binding. Hosted runtimes disable this after
1069    /// construction; ordinary local runtimes retain the compatible default.
1070    pub fn set_allow_local_workspaces(&self, allowed: bool) {
1071        self.allow_local_workspaces
1072            .store(allowed, Ordering::Relaxed);
1073    }
1074
1075    pub fn allows_local_workspaces(&self) -> bool {
1076        self.allow_local_workspaces.load(Ordering::Relaxed)
1077    }
1078
1079    pub async fn set_remote_runner_destination(&self, destination: Option<RunnerDestination>) {
1080        let lifecycle = destination.as_ref().map(|destination| RunnerLifecycle {
1081            destination_id: destination.id.clone(),
1082            provider_id: destination.provider_id.clone(),
1083            state: "configured".to_string(),
1084            session_id: None,
1085            timestamp: OffsetDateTime::now_utc(),
1086        });
1087        self.config.write().await.remote_runner_destination = destination;
1088        if let Some(lifecycle) = lifecycle {
1089            self.emit(RoderEvent::RunnerLifecycle(lifecycle)).await;
1090        } else {
1091            self.emit(RoderEvent::RunnerLifecycle(RunnerLifecycle {
1092                destination_id: "local".to_string(),
1093                provider_id: "local".to_string(),
1094                state: "local_fallback".to_string(),
1095                session_id: None,
1096                timestamp: OffsetDateTime::now_utc(),
1097            }))
1098            .await;
1099        }
1100    }
1101
1102    pub async fn set_file_backed_dynamic_context(&self, enabled: bool) -> RuntimeConfig {
1103        let mut cfg = self.config.write().await;
1104        cfg.file_backed_dynamic_context = enabled;
1105        cfg.clone()
1106    }
1107
1108    pub async fn set_command_shell(&self, shell: String) -> RuntimeConfig {
1109        let mut cfg = self.config.write().await;
1110        cfg.command_shell = shell;
1111        cfg.clone()
1112    }
1113
1114    pub async fn pending_plan_exit(&self) -> Option<PendingPlanExit> {
1115        let mut pending = self.pending_plan_exit.write().await;
1116        let current = pending.clone()?;
1117        if !current.is_expired(OffsetDateTime::now_utc()) {
1118            return Some(current);
1119        }
1120        *pending = None;
1121        drop(pending);
1122        self.emit_plan_exit_resolved(&current, false, self.status().await.policy_mode)
1123            .await;
1124        None
1125    }
1126
1127    pub async fn set_policy_mode(
1128        &self,
1129        mode: PolicyMode,
1130        reason: Option<String>,
1131    ) -> anyhow::Result<RuntimeConfig> {
1132        let mut cfg = self.config.write().await;
1133        let previous_mode = cfg.policy_mode;
1134        cfg.policy_mode = mode;
1135        let next = cfg.clone();
1136        drop(cfg);
1137        self.emit(RoderEvent::PolicyModeChanged(PolicyModeChanged {
1138            thread_id: "runtime".to_string(),
1139            turn_id: None,
1140            previous_mode,
1141            new_mode: mode,
1142            reason,
1143            timestamp: OffsetDateTime::now_utc(),
1144        }))
1145        .await;
1146        self.auto_resolve_pending_tool_approvals_for_mode(mode)
1147            .await;
1148        Ok(next)
1149    }
1150
1151    /// Toggle agent-swarm mode for the runtime (roadmap 104). When enabled, the
1152    /// swarm reminder is injected into each turn's developer instructions so
1153    /// every app-server/SDK client gets it, not only the TUI.
1154    pub async fn set_agent_swarm_mode(
1155        &self,
1156        enabled: bool,
1157        trigger: roder_api::subagents::AgentSwarmModeTrigger,
1158    ) -> anyhow::Result<RuntimeConfig> {
1159        let mut cfg = self.config.write().await;
1160        cfg.agent_swarm_mode = enabled;
1161        let next = cfg.clone();
1162        drop(cfg);
1163        self.emit(RoderEvent::AgentSwarmModeChanged(
1164            roder_api::subagents::AgentSwarmModeChanged {
1165                thread_id: "runtime".to_string(),
1166                turn_id: None,
1167                enabled,
1168                trigger,
1169                timestamp: OffsetDateTime::now_utc(),
1170            },
1171        ))
1172        .await;
1173        Ok(next)
1174    }
1175
1176    /// Toggle agent-swarm mode for a single thread (roadmap 104). The override is
1177    /// stored per thread so toggling swarm mode on one thread does not leak the
1178    /// reminder into other threads sharing the runtime; absent threads fall back
1179    /// to the runtime-global default. The emitted `AgentSwarmModeChanged` event
1180    /// carries the real `thread_id`.
1181    pub async fn set_agent_swarm_mode_for_thread(
1182        &self,
1183        thread_id: &str,
1184        enabled: bool,
1185        trigger: roder_api::subagents::AgentSwarmModeTrigger,
1186    ) -> bool {
1187        {
1188            let mut modes = self.agent_swarm_modes.write().await;
1189            modes.insert(thread_id.to_string(), enabled);
1190        }
1191        self.emit(RoderEvent::AgentSwarmModeChanged(
1192            roder_api::subagents::AgentSwarmModeChanged {
1193                thread_id: thread_id.to_string(),
1194                turn_id: None,
1195                enabled,
1196                trigger,
1197                timestamp: OffsetDateTime::now_utc(),
1198            },
1199        ))
1200        .await;
1201        enabled
1202    }
1203
1204    /// Resolve whether agent-swarm mode is active for `thread_id`: the per-thread
1205    /// override when present, otherwise the runtime-global default. The turn loop
1206    /// uses this to decide whether to inject the swarm reminder.
1207    pub async fn effective_agent_swarm_mode_for_thread(&self, thread_id: &str) -> bool {
1208        if let Some(enabled) = self.agent_swarm_modes.read().await.get(thread_id).copied() {
1209            return enabled;
1210        }
1211        self.status().await.agent_swarm_mode
1212    }
1213
1214    /// Toggle ultra mode for the runtime. When enabled, proactive multi-agent
1215    /// policy is injected into each turn's developer instructions for any
1216    /// model, so every app-server/SDK client gets it (not only the TUI).
1217    pub async fn set_ultra_mode(
1218        &self,
1219        enabled: bool,
1220        trigger: roder_api::subagents::UltraModeTrigger,
1221    ) -> anyhow::Result<RuntimeConfig> {
1222        let mut cfg = self.config.write().await;
1223        cfg.ultra_mode = enabled;
1224        let next = cfg.clone();
1225        drop(cfg);
1226        self.emit(RoderEvent::UltraModeChanged(
1227            roder_api::subagents::UltraModeChanged {
1228                thread_id: "runtime".to_string(),
1229                turn_id: None,
1230                enabled,
1231                trigger,
1232                timestamp: OffsetDateTime::now_utc(),
1233            },
1234        ))
1235        .await;
1236        Ok(next)
1237    }
1238
1239    /// Toggle ultra mode for a single thread. The override is stored per thread
1240    /// so toggling on one thread does not leak proactive multi-agent policy into
1241    /// other threads sharing the runtime; absent threads fall back to the
1242    /// runtime-global default.
1243    pub async fn set_ultra_mode_for_thread(
1244        &self,
1245        thread_id: &str,
1246        enabled: bool,
1247        trigger: roder_api::subagents::UltraModeTrigger,
1248    ) -> bool {
1249        {
1250            let mut modes = self.ultra_modes.write().await;
1251            modes.insert(thread_id.to_string(), enabled);
1252        }
1253        self.emit(RoderEvent::UltraModeChanged(
1254            roder_api::subagents::UltraModeChanged {
1255                thread_id: thread_id.to_string(),
1256                turn_id: None,
1257                enabled,
1258                trigger,
1259                timestamp: OffsetDateTime::now_utc(),
1260            },
1261        ))
1262        .await;
1263        enabled
1264    }
1265
1266    /// Resolve whether ultra mode is active for `thread_id`: the per-thread
1267    /// override when present, otherwise the runtime-global default. The turn
1268    /// loop uses this (together with Sol/Terra Ultra effort) to decide whether
1269    /// to inject proactive multi-agent policy.
1270    pub async fn effective_ultra_mode_for_thread(&self, thread_id: &str) -> bool {
1271        if let Some(enabled) = self.ultra_modes.read().await.get(thread_id).copied() {
1272            return enabled;
1273        }
1274        self.status().await.ultra_mode
1275    }
1276
1277    pub async fn set_hosted_web_search(
1278        &self,
1279        mode: HostedWebSearchMode,
1280    ) -> anyhow::Result<RuntimeConfig> {
1281        let mut cfg = self.config.write().await;
1282        cfg.hosted_web_search = HostedWebSearchConfig { mode };
1283        Ok(cfg.clone())
1284    }
1285
1286    async fn auto_resolve_pending_tool_approvals_for_mode(&self, mode: PolicyMode) {
1287        let gate = DefaultPolicyGate::new();
1288        let mut pending = self.pending_tool_approvals.lock().await;
1289        let approval_ids = pending
1290            .iter()
1291            .filter_map(|(approval_id, approval)| {
1292                let ctx = ToolExecutionContext::new(
1293                    approval.thread_id.clone(),
1294                    approval.turn_id.clone(),
1295                    mode,
1296                );
1297                matches!(
1298                    gate.decide(&approval.call, mode, &ctx),
1299                    PolicyDecision::AutoApproved { .. }
1300                )
1301                .then_some(approval_id.clone())
1302            })
1303            .collect::<Vec<_>>();
1304        let approvals = approval_ids
1305            .into_iter()
1306            .filter_map(|approval_id| {
1307                pending
1308                    .remove(&approval_id)
1309                    .map(|approval| (approval_id, approval))
1310            })
1311            .collect::<Vec<_>>();
1312        drop(pending);
1313
1314        for (approval_id, approval) in approvals {
1315            let ctx = ToolExecutionContext::new(
1316                approval.thread_id.clone(),
1317                approval.turn_id.clone(),
1318                mode,
1319            );
1320            let decision = gate.decide(&approval.call, mode, &ctx);
1321            self.emit(RoderEvent::PolicyDecisionRecorded(PolicyDecisionRecorded {
1322                thread_id: approval.thread_id.clone(),
1323                turn_id: approval.turn_id.clone(),
1324                tool_id: approval.tool_id.clone(),
1325                tool_name: approval.tool_name.clone(),
1326                mode,
1327                decision,
1328                timestamp: OffsetDateTime::now_utc(),
1329            }))
1330            .await;
1331            if mode == PolicyMode::Bypass {
1332                self.emit(RoderEvent::PolicyBypassActive(PolicyBypassActive {
1333                    thread_id: approval.thread_id.clone(),
1334                    turn_id: approval.turn_id.clone(),
1335                    tool_id: approval.tool_id.clone(),
1336                    tool_name: approval.tool_name.clone(),
1337                    timestamp: OffsetDateTime::now_utc(),
1338                }))
1339                .await;
1340            }
1341            self.emit(RoderEvent::ApprovalResolved(ApprovalResolved {
1342                thread_id: approval.thread_id,
1343                turn_id: approval.turn_id,
1344                approval_id,
1345                tool_id: approval.tool_id,
1346                tool_name: approval.tool_name,
1347                approved: true,
1348                timestamp: OffsetDateTime::now_utc(),
1349            }))
1350            .await;
1351            let _ = approval.tx.send(true);
1352        }
1353    }
1354
1355    pub async fn record_pending_plan_exit(&self, pending: PendingPlanExit) {
1356        *self.pending_plan_exit.write().await = Some(pending.clone());
1357        self.emit(RoderEvent::PolicyExitPlanRequested(
1358            PolicyExitPlanRequested {
1359                thread_id: pending.thread_id,
1360                turn_id: pending.turn_id,
1361                request_id: pending.request_id,
1362                target_mode: pending.target_mode,
1363                plan_summary: pending.plan_summary,
1364                next_steps: pending.next_steps,
1365                timestamp: OffsetDateTime::now_utc(),
1366            },
1367        ))
1368        .await;
1369    }
1370
1371    pub async fn resolve_pending_plan_exit(
1372        &self,
1373        request_id: &str,
1374        approved: bool,
1375    ) -> anyhow::Result<Option<PendingPlanExit>> {
1376        let mut pending = self.pending_plan_exit.write().await;
1377        let Some(current) = pending.clone() else {
1378            return Ok(None);
1379        };
1380        if current.request_id != request_id {
1381            anyhow::bail!("pending plan exit request {request_id:?} was not found");
1382        }
1383        *pending = None;
1384        drop(pending);
1385
1386        let approved = approved && !current.is_expired(OffsetDateTime::now_utc());
1387        let resolved_mode = if approved {
1388            let mut cfg = self.config.write().await;
1389            let previous_mode = cfg.policy_mode;
1390            cfg.policy_mode = current.target_mode;
1391            drop(cfg);
1392            self.emit(RoderEvent::PolicyModeChanged(PolicyModeChanged {
1393                thread_id: current.thread_id.clone(),
1394                turn_id: Some(current.turn_id.clone()),
1395                previous_mode,
1396                new_mode: current.target_mode,
1397                reason: Some("approved plan exit".to_string()),
1398                timestamp: OffsetDateTime::now_utc(),
1399            }))
1400            .await;
1401            self.auto_resolve_pending_tool_approvals_for_mode(current.target_mode)
1402                .await;
1403            current.target_mode
1404        } else {
1405            self.status().await.policy_mode
1406        };
1407        self.emit_plan_exit_resolved(&current, approved, resolved_mode)
1408            .await;
1409        Ok(Some(current))
1410    }
1411
1412    pub async fn resolve_tool_approval(
1413        &self,
1414        approval_id: &str,
1415        approved: bool,
1416    ) -> anyhow::Result<bool> {
1417        let pending = self.pending_tool_approvals.lock().await.remove(approval_id);
1418        let Some(pending) = pending else {
1419            return Ok(false);
1420        };
1421        self.emit(RoderEvent::ApprovalResolved(ApprovalResolved {
1422            thread_id: pending.thread_id,
1423            turn_id: pending.turn_id,
1424            approval_id: approval_id.to_string(),
1425            tool_id: pending.tool_id,
1426            tool_name: pending.tool_name,
1427            approved,
1428            timestamp: OffsetDateTime::now_utc(),
1429        }))
1430        .await;
1431        let _ = pending.tx.send(approved);
1432        Ok(true)
1433    }
1434
1435    pub async fn request_app_server_tool_approval(
1436        &self,
1437        call: ToolCall,
1438        reason: Option<String>,
1439    ) -> anyhow::Result<bool> {
1440        let approval_id = call.id.clone();
1441        let (tx, rx) = oneshot::channel();
1442        self.pending_tool_approvals.lock().await.insert(
1443            approval_id.clone(),
1444            PendingToolApproval {
1445                thread_id: call.thread_id.clone(),
1446                turn_id: call.turn_id.clone(),
1447                tool_id: call.id.clone(),
1448                tool_name: call.name.clone(),
1449                call: call.clone(),
1450                tx,
1451            },
1452        );
1453        self.emit(RoderEvent::ApprovalRequested(ApprovalRequested {
1454            thread_id: call.thread_id.clone(),
1455            turn_id: call.turn_id.clone(),
1456            approval_id,
1457            tool_id: call.id.clone(),
1458            tool_name: call.name.clone(),
1459            reason,
1460            timestamp: OffsetDateTime::now_utc(),
1461        }))
1462        .await;
1463        Ok(rx.await.unwrap_or(false))
1464    }
1465
1466    /// Completes a pending host-executed tool call (`tools/resolve`). Returns false when the
1467    /// request id is unknown, already resolved, timed out, or cancelled by a turn interrupt.
1468    pub async fn resolve_external_tool_call(
1469        &self,
1470        request_id: &str,
1471        resolution: ExternalToolResolution,
1472    ) -> anyhow::Result<bool> {
1473        let pending = self
1474            .pending_external_tool_calls
1475            .lock()
1476            .await
1477            .remove(request_id);
1478        let Some(pending) = pending else {
1479            return Ok(false);
1480        };
1481        self.emit(RoderEvent::ExternalToolCallResolved(
1482            ExternalToolCallResolved {
1483                thread_id: pending.thread_id,
1484                turn_id: pending.turn_id,
1485                request_id: request_id.to_string(),
1486                tool_id: pending.tool_id,
1487                tool_name: pending.tool_name,
1488                outcome: ExternalToolCallOutcome::Resolved,
1489                is_error: resolution.is_error,
1490                timestamp: OffsetDateTime::now_utc(),
1491            },
1492        ))
1493        .await;
1494        let _ = pending.tx.send(resolution);
1495        Ok(true)
1496    }
1497
1498    /// Drops every pending external tool call for the turn and reports them as cancelled so an
1499    /// interrupt does not leak requests waiting on `tools/resolve`.
1500    async fn cancel_pending_external_tool_calls_for_turn(&self, turn_id: &TurnId) {
1501        let cancelled = {
1502            let mut pending = self.pending_external_tool_calls.lock().await;
1503            let request_ids = pending
1504                .iter()
1505                .filter(|(_, call)| &call.turn_id == turn_id)
1506                .map(|(request_id, _)| request_id.clone())
1507                .collect::<Vec<_>>();
1508            request_ids
1509                .into_iter()
1510                .filter_map(|request_id| pending.remove(&request_id).map(|call| (request_id, call)))
1511                .collect::<Vec<_>>()
1512        };
1513        for (request_id, call) in cancelled {
1514            self.emit(RoderEvent::ExternalToolCallResolved(
1515                ExternalToolCallResolved {
1516                    thread_id: call.thread_id,
1517                    turn_id: call.turn_id,
1518                    request_id,
1519                    tool_id: call.tool_id,
1520                    tool_name: call.tool_name,
1521                    outcome: ExternalToolCallOutcome::Cancelled,
1522                    is_error: true,
1523                    timestamp: OffsetDateTime::now_utc(),
1524                },
1525            ))
1526            .await;
1527        }
1528    }
1529
1530    pub async fn resolve_user_input(
1531        &self,
1532        request_id: &str,
1533        answers: serde_json::Value,
1534    ) -> anyhow::Result<bool> {
1535        let pending = self.pending_user_inputs.lock().await.remove(request_id);
1536        let Some(pending) = pending else {
1537            return Ok(false);
1538        };
1539        self.emit(RoderEvent::UserInputResolved(UserInputResolved {
1540            thread_id: pending.thread_id,
1541            turn_id: pending.turn_id,
1542            request_id: request_id.to_string(),
1543            answers: answers.clone(),
1544            timestamp: OffsetDateTime::now_utc(),
1545        }))
1546        .await;
1547        let _ = pending.tx.send(answers);
1548        Ok(true)
1549    }
1550
1551    async fn emit_plan_exit_resolved(
1552        &self,
1553        current: &PendingPlanExit,
1554        approved: bool,
1555        resolved_mode: PolicyMode,
1556    ) {
1557        self.emit(RoderEvent::PolicyExitPlanResolved(PolicyExitPlanResolved {
1558            thread_id: current.thread_id.clone(),
1559            turn_id: current.turn_id.clone(),
1560            request_id: current.request_id.clone(),
1561            approved,
1562            target_mode: current.target_mode,
1563            resolved_mode,
1564            timestamp: OffsetDateTime::now_utc(),
1565        }))
1566        .await;
1567    }
1568
1569    pub async fn select_provider(
1570        &self,
1571        provider: String,
1572        model: Option<String>,
1573        reasoning: Option<String>,
1574    ) -> anyhow::Result<RuntimeConfig> {
1575        let next = self
1576            .preview_provider_selection(provider, model, reasoning)
1577            .await?;
1578        let mut cfg = self.config.write().await;
1579        *cfg = next;
1580        Ok(cfg.clone())
1581    }
1582
1583    pub async fn preview_provider_selection(
1584        &self,
1585        provider: String,
1586        model: Option<String>,
1587        reasoning: Option<String>,
1588    ) -> anyhow::Result<RuntimeConfig> {
1589        self.engine_for(&provider)?;
1590        let mut cfg = self.config.read().await.clone();
1591        cfg.default_provider = provider;
1592        if let Some(model) = model {
1593            cfg.default_model = model;
1594        }
1595        if let Some(reasoning) = reasoning {
1596            if reasoning == REASONING_NONE
1597                && !model_supports_reasoning(&cfg.default_model, &reasoning)
1598            {
1599                return Ok(cfg.clone());
1600            }
1601            validate_reasoning_effort(&cfg.default_model, &reasoning)?;
1602            cfg.reasoning = Some(reasoning);
1603        }
1604        Ok(cfg)
1605    }
1606
1607    pub async fn effective_reasoning(&self) -> String {
1608        let cfg = self.config.read().await;
1609        effective_reasoning_for_model(&cfg, &cfg.default_model)
1610    }
1611
1612    pub fn effective_reasoning_for_config(cfg: &RuntimeConfig) -> String {
1613        effective_reasoning_for_model(cfg, &cfg.default_model)
1614    }
1615
1616    pub async fn set_dynamic_workflow_effort(
1617        &self,
1618        effort_profile: DynamicWorkflowEffortProfile,
1619    ) -> RuntimeConfig {
1620        let mut cfg = self.config.write().await;
1621        cfg.dynamic_workflows.effort_profile = effort_profile;
1622        cfg.clone()
1623    }
1624
1625    pub async fn dynamic_workflow_trigger_decision(
1626        &self,
1627        message: &str,
1628    ) -> WorkflowTriggerDecision {
1629        let cfg = self.config.read().await;
1630        classify_workflow_trigger(message, &cfg.dynamic_workflows)
1631    }
1632
1633    pub async fn create_thread(&self, title: Option<String>) -> anyhow::Result<ThreadMetadata> {
1634        self.create_thread_with(CreateThreadRequest {
1635            title,
1636            workspace: self.workspace.display().to_string(),
1637            workspace_id: None,
1638            root_id: None,
1639            provider: None,
1640            model: None,
1641            selection_mode: None,
1642            tool_allowlist: Vec::new(),
1643            developer_instructions: None,
1644            external_tools: Vec::new(),
1645            runner: None,
1646        })
1647        .await
1648    }
1649
1650    /**
1651     * Resolves an explicit thread-runner selection into a persisted binding.
1652     * Fails fast at thread creation when the provider is missing or rejects
1653     * the destination, instead of surfacing the error on the first tool call.
1654     */
1655    async fn resolve_thread_runner_binding(
1656        &self,
1657        thread_id: &str,
1658        selection: ThreadRunnerSelection,
1659    ) -> anyhow::Result<ThreadRunnerBinding> {
1660        let provider = self
1661            .registry
1662            .remote_runner_providers
1663            .iter()
1664            .find(|provider| provider.id() == selection.provider_id)
1665            .cloned()
1666            .ok_or_else(|| {
1667                anyhow::anyhow!(
1668                    "remote runner provider {:?} is not installed",
1669                    selection.provider_id
1670                )
1671            })?;
1672        let workspace = selection.workspace.trim();
1673        anyhow::ensure!(
1674            std::path::Path::new(workspace).is_absolute(),
1675            "runner workspace must be an absolute path on the runner: {workspace:?}"
1676        );
1677        let mut read_roots = Vec::with_capacity(selection.read_roots.len());
1678        for read_root in &selection.read_roots {
1679            let trimmed = read_root.trim();
1680            anyhow::ensure!(
1681                std::path::Path::new(trimmed).is_absolute(),
1682                "runner read root must be an absolute path on the runner: {trimmed:?}"
1683            );
1684            read_roots.push(PathBuf::from(trimmed));
1685        }
1686        let destination = RunnerDestination {
1687            id: format!("thread-{thread_id}"),
1688            provider_id: selection.provider_id,
1689            config: selection.config,
1690            default_manifest: roder_api::remote_runner::RunnerManifest::default(),
1691        };
1692        provider.validate_destination(&destination).await?;
1693        Ok(ThreadRunnerBinding {
1694            destination,
1695            workspace: PathBuf::from(workspace),
1696            read_roots,
1697        })
1698    }
1699
1700    /**
1701     * Build a per-thread binding from a runtime-level destination (selected via
1702     * the TUI runner picker or config `default_destination`) when the
1703     * destination's provider advertises a default workspace. Returns `None` when
1704     * there is no destination or the provider opts out (no default workspace),
1705     * preserving the legacy behavior where such threads keep local tools.
1706     */
1707    async fn synthesize_runtime_runner_binding(
1708        &self,
1709        thread_id: &str,
1710        destination: Option<RunnerDestination>,
1711    ) -> anyhow::Result<Option<ThreadRunnerBinding>> {
1712        let Some(destination) = destination else {
1713            return Ok(None);
1714        };
1715        let Some(provider) = self
1716            .registry
1717            .remote_runner_providers
1718            .iter()
1719            .find(|provider| provider.id() == destination.provider_id)
1720        else {
1721            return Ok(None);
1722        };
1723        // An explicit workspace in the destination config wins over the
1724        // provider default; absence of both means the provider opts out.
1725        let workspace = destination
1726            .config
1727            .get("working_dir")
1728            .and_then(serde_json::Value::as_str)
1729            .map(str::to_string)
1730            .or_else(|| provider.default_workspace());
1731        let Some(workspace) = workspace else {
1732            return Ok(None);
1733        };
1734        let selection = ThreadRunnerSelection {
1735            provider_id: destination.provider_id.clone(),
1736            config: destination.config.clone(),
1737            workspace,
1738            read_roots: Vec::new(),
1739        };
1740        Ok(Some(
1741            self.resolve_thread_runner_binding(thread_id, selection)
1742                .await?,
1743        ))
1744    }
1745
1746    /// Validates a runner selection without creating a thread; the placeholder destination id only appears in error messages.
1747    pub async fn validate_thread_runner_selection(
1748        &self,
1749        selection: ThreadRunnerSelection,
1750    ) -> anyhow::Result<()> {
1751        self.resolve_thread_runner_binding("validate", selection)
1752            .await
1753            .map(|_| ())
1754    }
1755
1756    pub async fn create_thread_with(
1757        &self,
1758        req: CreateThreadRequest,
1759    ) -> anyhow::Result<ThreadMetadata> {
1760        let cfg = self.config.read().await.clone();
1761        let now = OffsetDateTime::now_utc();
1762        let workspace = validate_thread_workspace(&req.workspace)?;
1763        let provider = req.provider.unwrap_or(cfg.default_provider);
1764        let model = req.model.unwrap_or(cfg.default_model);
1765        let selection_mode = req
1766            .selection_mode
1767            .unwrap_or_else(|| ModelSelectionMode::manual(provider.clone(), model.clone(), None));
1768        let thread_id = uuid::Uuid::new_v4().to_string();
1769        let runner_binding = match req.runner {
1770            Some(selection) => Some(
1771                self.resolve_thread_runner_binding(&thread_id, selection)
1772                    .await?,
1773            ),
1774            // No explicit per-thread selection: if a runtime-level destination
1775            // is active and its provider advertises a default workspace, bind
1776            // the new thread so its coding tools route into the runner. This is
1777            // what makes a TUI/config-selected runner (e.g. Blaxel) actually
1778            // execute tools in the sandbox instead of locally. Providers that
1779            // return no default workspace stay unbound (legacy local-tools).
1780            None => {
1781                self.synthesize_runtime_runner_binding(
1782                    &thread_id,
1783                    cfg.remote_runner_destination.clone(),
1784                )
1785                .await?
1786            }
1787        };
1788        let runner_destination = runner_binding
1789            .as_ref()
1790            .map(|binding| binding.destination.clone())
1791            .or_else(|| cfg.remote_runner_destination.clone());
1792        let metadata = ThreadMetadata {
1793            thread_id,
1794            title: req.title,
1795            workspace,
1796            workspace_id: req.workspace_id,
1797            root_id: req.root_id,
1798            provider: Some(provider),
1799            model: Some(model),
1800            selection_mode: Some(selection_mode),
1801            tool_allowlist: req.tool_allowlist,
1802            developer_instructions: req.developer_instructions,
1803            external_tools: req.external_tools,
1804            runner_destination,
1805            runner_state: None,
1806            runner_binding,
1807            created_at: now,
1808            updated_at: now,
1809            message_count: 0,
1810            usage: None,
1811            parent_thread_id: None,
1812            forked_from_turn_id: None,
1813            workspace_fork: None,
1814        };
1815
1816        let metadata = if let Some(store) = &self.thread_store {
1817            store.create_thread(metadata).await?
1818        } else {
1819            metadata
1820        };
1821        self.emit(RoderEvent::ThreadCreated(ThreadCreated {
1822            thread_id: metadata.thread_id.clone(),
1823            timestamp: OffsetDateTime::now_utc(),
1824        }))
1825        .await;
1826        Ok(metadata)
1827    }
1828
1829    pub async fn list_threads(&self) -> anyhow::Result<Vec<ThreadMetadata>> {
1830        if let Some(store) = &self.thread_store {
1831            return store.list_threads().await;
1832        }
1833        Ok(Vec::new())
1834    }
1835
1836    pub async fn list_threads_page(
1837        &self,
1838        options: roder_api::thread::ThreadListOptions,
1839    ) -> anyhow::Result<roder_api::thread::ThreadListPage> {
1840        if let Some(store) = &self.thread_store {
1841            return store.list_threads_page(options).await;
1842        }
1843        Ok(roder_api::thread::ThreadListPage::default())
1844    }
1845
1846    pub async fn load_thread_metadata(
1847        &self,
1848        thread_id: &str,
1849    ) -> anyhow::Result<Option<roder_api::thread::ThreadMetadata>> {
1850        if let Some(store) = &self.thread_store {
1851            return store.load_thread_metadata(&thread_id.to_string()).await;
1852        }
1853        Ok(None)
1854    }
1855
1856    pub async fn archive_thread(&self, thread_id: &str) -> anyhow::Result<bool> {
1857        let archived = if let Some(store) = &self.thread_store {
1858            store.archive_thread(&thread_id.to_string()).await?
1859        } else {
1860            false
1861        };
1862        if archived {
1863            self.thread_item_cache
1864                .lock()
1865                .await
1866                .remove_thread(&thread_id.to_string());
1867            self.evict_runner_session(&thread_id.to_string()).await;
1868        }
1869        Ok(archived)
1870    }
1871
1872    pub async fn start_team(&self, req: TeamStartRequest) -> anyhow::Result<TeamState> {
1873        let cfg = self.config.read().await.clone();
1874        let workspace = self.workspace.display().to_string();
1875        let lead_thread_id = match req.lead_thread_id {
1876            Some(thread_id) => thread_id,
1877            None => {
1878                self.create_thread_with(CreateThreadRequest {
1879                    title: Some("Team lead".to_string()),
1880                    workspace: workspace.clone(),
1881                    workspace_id: None,
1882                    root_id: None,
1883                    provider: None,
1884                    model: None,
1885                    selection_mode: None,
1886                    tool_allowlist: Vec::new(),
1887                    developer_instructions: None,
1888                    external_tools: Vec::new(),
1889                    runner: None,
1890                })
1891                .await?
1892                .thread_id
1893            }
1894        };
1895        let team_id = uuid::Uuid::new_v4().to_string();
1896        let active_lead_turn_id = self.active_turn_for_thread(&lead_thread_id).await;
1897        let lead_selection = match active_lead_turn_id.as_ref() {
1898            Some(turn_id) => self
1899                .active_turn_selections
1900                .read()
1901                .await
1902                .get(turn_id)
1903                .cloned(),
1904            None => self.selection_mode_for_thread(&lead_thread_id).await?,
1905        };
1906        let lead_concrete_selection = lead_selection
1907            .as_ref()
1908            .map(ModelSelectionMode::concrete_selection);
1909        let mut lead = crate::teams::lead_member(
1910            lead_thread_id.clone(),
1911            lead_concrete_selection
1912                .as_ref()
1913                .map(|selection| selection.provider.clone())
1914                .or_else(|| Some(cfg.default_provider.clone())),
1915            lead_concrete_selection
1916                .as_ref()
1917                .map(|selection| selection.model.clone())
1918                .or_else(|| Some(cfg.default_model.clone())),
1919            cfg.policy_mode,
1920        );
1921        if let Some(turn_id) = active_lead_turn_id {
1922            lead.current_turn_id = Some(turn_id);
1923            lead.status = TeamMemberStatus::Running;
1924        }
1925        let mut members = vec![lead];
1926
1927        for (index, member) in req.members.into_iter().enumerate() {
1928            let thread = self
1929                .create_thread_with(CreateThreadRequest {
1930                    title: Some(member.name.clone()),
1931                    workspace: workspace.clone(),
1932                    workspace_id: None,
1933                    root_id: None,
1934                    provider: member.model_provider.clone(),
1935                    model: member.model.clone(),
1936                    selection_mode: None,
1937                    tool_allowlist: Vec::new(),
1938                    developer_instructions: None,
1939                    external_tools: Vec::new(),
1940                    runner: None,
1941                })
1942                .await?;
1943            let member_id = format!("member-{}", index + 1);
1944            let descriptor = crate::teams::teammate_member(
1945                member_id.clone(),
1946                member.name,
1947                thread.thread_id.clone(),
1948                member.model_provider.or(thread.provider),
1949                member.model.or(thread.model),
1950                cfg.policy_mode,
1951            );
1952            members.push(descriptor);
1953        }
1954
1955        let now = OffsetDateTime::now_utc();
1956        let team = self
1957            .teams
1958            .insert(TeamState {
1959                id: team_id.clone(),
1960                lead_thread_id: lead_thread_id.clone(),
1961                display_mode: req.display_mode,
1962                members,
1963                mailbox: Vec::new(),
1964                tasks: Vec::new(),
1965                created_at: now,
1966                updated_at: now,
1967            })
1968            .await?;
1969        self.emit(RoderEvent::TeamStarted(TeamStarted {
1970            team_id: team_id.clone(),
1971            lead_thread_id,
1972            display_mode: team.display_mode,
1973            members: team.members.clone(),
1974            tasks: team.tasks.clone(),
1975            timestamp: OffsetDateTime::now_utc(),
1976        }))
1977        .await;
1978        for member in team
1979            .members
1980            .iter()
1981            .filter(|member| member.role != roder_api::teams::TeamMemberRole::Lead)
1982        {
1983            self.emit(RoderEvent::TeamMemberStarted(TeamMemberStarted {
1984                team_id: team_id.clone(),
1985                member: member.clone(),
1986                timestamp: OffsetDateTime::now_utc(),
1987            }))
1988            .await;
1989        }
1990        Ok(team)
1991    }
1992
1993    pub async fn list_teams(&self) -> Vec<TeamState> {
1994        let active_thread_ids = self
1995            .active_turns
1996            .read()
1997            .await
1998            .values()
1999            .map(|handle| handle.thread_id.clone())
2000            .collect::<std::collections::HashSet<_>>();
2001        self.teams
2002            .list()
2003            .await
2004            .into_iter()
2005            .map(|team| runtime_local_team_view(team, &active_thread_ids))
2006            .collect()
2007    }
2008
2009    pub async fn read_team(&self, team_id: &str) -> Option<TeamState> {
2010        let active_thread_ids = self
2011            .active_turns
2012            .read()
2013            .await
2014            .values()
2015            .map(|handle| handle.thread_id.clone())
2016            .collect::<std::collections::HashSet<_>>();
2017        self.teams
2018            .get(team_id)
2019            .await
2020            .map(|team| runtime_local_team_view(team, &active_thread_ids))
2021    }
2022
2023    pub async fn start_team_member(
2024        &self,
2025        team_id: &str,
2026        req: TeamMemberStartRequest,
2027    ) -> anyhow::Result<TeamState> {
2028        self.start_team_member_with_selection(team_id, req, None)
2029            .await
2030    }
2031
2032    pub async fn message_team_member(
2033        self: &Arc<Self>,
2034        team_id: &str,
2035        member_id: &str,
2036        message: String,
2037    ) -> anyhow::Result<TurnId> {
2038        let team = self
2039            .read_team(team_id)
2040            .await
2041            .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2042        self.followup_team_member(&team.lead_thread_id, team_id, member_id, message)
2043            .await
2044    }
2045
2046    /// Queue a mailbox message without starting an idle agent. If the target is
2047    /// already running, the message is delivered at the next inference boundary.
2048    pub(crate) async fn queue_team_member_message(
2049        self: &Arc<Self>,
2050        caller_thread_id: &ThreadId,
2051        team_id: &str,
2052        member_id: &str,
2053        message: String,
2054    ) -> anyhow::Result<Option<TurnId>> {
2055        let _delivery_guard = self.agent_team_spawn_lock.lock().await;
2056        let team = self
2057            .read_team(team_id)
2058            .await
2059            .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2060        let member = team
2061            .members
2062            .iter()
2063            .find(|member| member.id == member_id)
2064            .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2065            .clone();
2066        if member.status == TeamMemberStatus::Closed {
2067            anyhow::bail!("subagent {} is closed", member.name);
2068        }
2069        let from_member_id = team
2070            .members
2071            .iter()
2072            .find(|candidate| candidate.thread_id == *caller_thread_id)
2073            .map(|candidate| candidate.id.clone());
2074        self.teams
2075            .append_mailbox_message(
2076                team_id,
2077                from_member_id,
2078                member_id.to_string(),
2079                TeamMailboxMessageKind::Message,
2080                message,
2081            )
2082            .await?;
2083        let Some(turn_id) = self.active_turn_for_thread(&member.thread_id).await else {
2084            return Ok(None);
2085        };
2086        self.deliver_pending_team_mailbox(team_id, member_id, member.thread_id, turn_id.clone())
2087            .await?;
2088        Ok(Some(turn_id))
2089    }
2090
2091    /// Deliver queued messages and start an idle agent, or steer its active turn.
2092    pub(crate) async fn followup_team_member(
2093        self: &Arc<Self>,
2094        caller_thread_id: &ThreadId,
2095        team_id: &str,
2096        member_id: &str,
2097        message: String,
2098    ) -> anyhow::Result<TurnId> {
2099        let _spawn_guard = self.agent_team_spawn_lock.lock().await;
2100        self.followup_team_member_locked(caller_thread_id, team_id, member_id, message)
2101            .await
2102    }
2103
2104    async fn followup_team_member_locked(
2105        self: &Arc<Self>,
2106        caller_thread_id: &ThreadId,
2107        team_id: &str,
2108        member_id: &str,
2109        message: String,
2110    ) -> anyhow::Result<TurnId> {
2111        let team = self
2112            .read_team(team_id)
2113            .await
2114            .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2115        let member = team
2116            .members
2117            .iter()
2118            .find(|member| member.id == member_id)
2119            .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2120            .clone();
2121        if member.status == TeamMemberStatus::Closed {
2122            anyhow::bail!("subagent {} is closed", member.name);
2123        }
2124        let from_member_id = team
2125            .members
2126            .iter()
2127            .find(|candidate| candidate.thread_id == *caller_thread_id)
2128            .map(|candidate| candidate.id.clone());
2129        self.teams
2130            .append_mailbox_message(
2131                team_id,
2132                from_member_id,
2133                member_id.to_string(),
2134                TeamMailboxMessageKind::NewTask,
2135                message,
2136            )
2137            .await?;
2138        if let Some(turn_id) = self.active_turn_for_thread(&member.thread_id).await {
2139            self.deliver_pending_team_mailbox(
2140                team_id,
2141                member_id,
2142                member.thread_id,
2143                turn_id.clone(),
2144            )
2145            .await?;
2146            return Ok(turn_id);
2147        }
2148
2149        self.ensure_codex_v2_team_capacity(&team, &member.id, None)
2150            .await?;
2151        let metadata = self.load_thread_metadata(&member.thread_id).await?;
2152        let inherited_context = self
2153            .team_member_turn_contexts
2154            .lock()
2155            .await
2156            .get(&member.thread_id)
2157            .cloned();
2158        let workspace = inherited_context
2159            .as_ref()
2160            .map(|context| context.workspace.clone())
2161            .or_else(|| metadata.as_ref().map(|metadata| metadata.workspace.clone()))
2162            .unwrap_or_else(|| self.workspace.display().to_string());
2163        let member_thread_id = member.thread_id;
2164        let turn_id = self
2165            .start_turn(StartTurnRequest {
2166                thread_id: member_thread_id.clone(),
2167                message: String::new(),
2168                images: Vec::new(),
2169                provider_override: member.model_provider,
2170                model_override: member.model,
2171                reasoning_override: metadata
2172                    .as_ref()
2173                    .and_then(|metadata| metadata.selection_mode.as_ref())
2174                    .and_then(ModelSelectionMode::reasoning)
2175                    .map(str::to_string),
2176                workspace,
2177                instructions: inherited_context
2178                    .as_ref()
2179                    .map(|context| context.instructions.clone())
2180                    .unwrap_or_else(crate::default_instructions),
2181                developer_context: inherited_context
2182                    .as_ref()
2183                    .and_then(|context| context.developer_context.clone()),
2184                task_ledger_required: false,
2185            })
2186            .await?;
2187        Ok(turn_id)
2188    }
2189
2190    pub async fn set_team_member_policy_mode(
2191        &self,
2192        team_id: &str,
2193        member_id: &str,
2194        policy_mode: PolicyMode,
2195    ) -> anyhow::Result<TeamState> {
2196        self.teams
2197            .set_member_policy_mode(team_id, member_id, policy_mode)
2198            .await
2199    }
2200
2201    pub async fn interrupt_team_member(
2202        &self,
2203        team_id: &str,
2204        member_id: &str,
2205    ) -> anyhow::Result<Option<TurnId>> {
2206        let _interrupt_guard = self.agent_team_spawn_lock.lock().await;
2207        let team = self
2208            .read_team(team_id)
2209            .await
2210            .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2211        let member = team
2212            .members
2213            .iter()
2214            .find(|member| member.id == member_id)
2215            .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2216            .clone();
2217        if member.status != TeamMemberStatus::Running {
2218            return Ok(None);
2219        }
2220        let Some(turn_id) = member.current_turn_id.clone() else {
2221            return Ok(None);
2222        };
2223        if self
2224            .active_turn_for_thread(&member.thread_id)
2225            .await
2226            .as_ref()
2227            != Some(&turn_id)
2228        {
2229            return Ok(None);
2230        }
2231        self.interrupt_turn(member.thread_id.clone(), turn_id.clone())
2232            .await?;
2233        // Make queued mailbox work immediately eligible for the replacement turn while the
2234        // lifecycle lock still prevents a follow-up from starting. Delivery acknowledgements
2235        // are turn-owned, so a late acknowledgement from the aborted turn cannot steal a
2236        // reservation acquired by its replacement.
2237        self.teams
2238            .release_mailbox_reservations_for_turn(&turn_id)
2239            .await;
2240        self.teams
2241            .update_member(team_id, member_id, |member| {
2242                member.status = TeamMemberStatus::Interrupted;
2243                member.current_turn_id = None;
2244                member.terminal_error = None;
2245            })
2246            .await?;
2247        self.emit(RoderEvent::TeamMemberCompleted(TeamMemberCompleted {
2248            team_id: team_id.to_string(),
2249            member_id: member_id.to_string(),
2250            member_thread_id: member.thread_id,
2251            turn_id: Some(turn_id.clone()),
2252            status: TeamMemberStatus::Interrupted,
2253            final_message: member.final_message,
2254            error: None,
2255            timestamp: OffsetDateTime::now_utc(),
2256        }))
2257        .await;
2258        Ok(Some(turn_id))
2259    }
2260
2261    pub async fn close_team_member(
2262        &self,
2263        team_id: &str,
2264        member_id: &str,
2265    ) -> anyhow::Result<roder_api::teams::TeamMemberDescriptor> {
2266        let team = self
2267            .read_team(team_id)
2268            .await
2269            .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2270        let member = team
2271            .members
2272            .iter()
2273            .find(|member| member.id == member_id)
2274            .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2275            .clone();
2276        if member.role == roder_api::teams::TeamMemberRole::Lead {
2277            anyhow::bail!("team lead cannot be closed as a subagent");
2278        }
2279        let interrupted_turn_id = if member.status == TeamMemberStatus::Running {
2280            if let Some(turn_id) = member.current_turn_id.clone() {
2281                self.interrupt_turn(member.thread_id.clone(), turn_id.clone())
2282                    .await?;
2283                Some(turn_id)
2284            } else {
2285                None
2286            }
2287        } else {
2288            member.current_turn_id.clone()
2289        };
2290        let updated = self
2291            .teams
2292            .update_member(team_id, member_id, |member| {
2293                member.status = TeamMemberStatus::Closed;
2294                member.current_turn_id = None;
2295            })
2296            .await?;
2297        let closed = updated
2298            .members
2299            .iter()
2300            .find(|member| member.id == member_id)
2301            .cloned()
2302            .ok_or_else(|| anyhow::anyhow!("closed team member disappeared"))?;
2303        self.team_member_turn_contexts
2304            .lock()
2305            .await
2306            .remove(&closed.thread_id);
2307        self.emit(RoderEvent::TeamMemberCompleted(TeamMemberCompleted {
2308            team_id: team_id.to_string(),
2309            member_id: closed.id.clone(),
2310            member_thread_id: closed.thread_id.clone(),
2311            turn_id: interrupted_turn_id,
2312            status: TeamMemberStatus::Closed,
2313            final_message: closed.final_message.clone(),
2314            error: closed.terminal_error.clone(),
2315            timestamp: OffsetDateTime::now_utc(),
2316        }))
2317        .await;
2318        Ok(closed)
2319    }
2320
2321    pub async fn cleanup_team(&self, team_id: &str, force: bool) -> anyhow::Result<bool> {
2322        let Some(team) = self.read_team(team_id).await else {
2323            return Ok(false);
2324        };
2325        if !force
2326            && team
2327                .members
2328                .iter()
2329                .any(|member| member.status == TeamMemberStatus::Running)
2330        {
2331            anyhow::bail!("team {team_id:?} has active teammates; use forced cleanup");
2332        }
2333        if force {
2334            for member in team.members.iter().filter(|member| {
2335                member.role != roder_api::teams::TeamMemberRole::Lead
2336                    && member.status == TeamMemberStatus::Running
2337            }) {
2338                if let Some(turn_id) = member
2339                    .current_turn_id
2340                    .clone()
2341                    .or(self.active_turn_for_thread(&member.thread_id).await)
2342                {
2343                    let _ = self.interrupt_turn(member.thread_id.clone(), turn_id).await;
2344                }
2345            }
2346        }
2347        let removed = self.teams.remove(team_id).await?.is_some();
2348        if removed {
2349            let member_thread_ids = team
2350                .members
2351                .iter()
2352                .map(|member| member.thread_id.clone())
2353                .collect::<std::collections::HashSet<_>>();
2354            self.team_member_turn_contexts
2355                .lock()
2356                .await
2357                .retain(|thread_id, _| !member_thread_ids.contains(thread_id));
2358            self.emit(RoderEvent::TeamCleanupCompleted(TeamCleanupCompleted {
2359                team_id: team_id.to_string(),
2360                forced: force,
2361                timestamp: OffsetDateTime::now_utc(),
2362            }))
2363            .await;
2364        }
2365        Ok(removed)
2366    }
2367
2368    pub async fn effective_policy_mode_for_thread(&self, thread_id: &str) -> PolicyMode {
2369        if let Some(mode) = self.teams.policy_mode_for_thread(thread_id).await {
2370            return mode;
2371        }
2372        self.status().await.policy_mode
2373    }
2374
2375    async fn complete_team_member_turn_with_result(
2376        &self,
2377        thread_id: &ThreadId,
2378        turn_id: &TurnId,
2379        status: TeamMemberStatus,
2380        final_message: Option<String>,
2381        terminal_error: Option<String>,
2382    ) -> anyhow::Result<()> {
2383        let _delivery_guard = self.agent_team_spawn_lock.lock().await;
2384        let Some((team_id, member)) = self
2385            .teams
2386            .complete_member_turn(
2387                thread_id,
2388                turn_id,
2389                status,
2390                final_message.clone(),
2391                terminal_error.clone(),
2392            )
2393            .await?
2394        else {
2395            return Ok(());
2396        };
2397        if let Some(parent_thread_id) = member.parent_thread_id.as_ref()
2398            && let Some(team) = self.read_team(&team_id).await
2399            && let Some(parent) = team
2400                .members
2401                .iter()
2402                .find(|candidate| candidate.thread_id == *parent_thread_id)
2403        {
2404            let identity = member
2405                .agent_path
2406                .as_deref()
2407                .or(member.task_name.as_deref())
2408                .unwrap_or(&member.name);
2409            let mut report = format!("Agent {identity} finished with status {status:?}.");
2410            if let Some(message) = final_message.as_deref()
2411                && !message.trim().is_empty()
2412            {
2413                report.push_str("\n\nFinal result:\n");
2414                report.push_str(message);
2415            }
2416            if let Some(error) = terminal_error.as_deref()
2417                && !error.trim().is_empty()
2418            {
2419                report.push_str("\n\nTerminal error:\n");
2420                report.push_str(error);
2421            }
2422            let mailbox_appended = self
2423                .teams
2424                .append_mailbox_message(
2425                    &team_id,
2426                    Some(member.id.clone()),
2427                    parent.id.clone(),
2428                    TeamMailboxMessageKind::FinalAnswer,
2429                    report,
2430                )
2431                .await
2432                .is_ok();
2433            if mailbox_appended
2434                && let Some(parent_turn_id) = self.active_turn_for_thread(parent_thread_id).await
2435            {
2436                // The parent may finish between the active-turn lookup and enqueue. Its durable
2437                // mailbox entry remains pending for the next turn, while terminal observers must
2438                // still receive TeamMemberCompleted for this child.
2439                let _ = self
2440                    .deliver_pending_team_mailbox(
2441                        &team_id,
2442                        &parent.id,
2443                        parent_thread_id.clone(),
2444                        parent_turn_id,
2445                    )
2446                    .await;
2447            }
2448        }
2449        self.emit(RoderEvent::TeamMemberCompleted(TeamMemberCompleted {
2450            team_id,
2451            member_id: member.id,
2452            member_thread_id: member.thread_id,
2453            turn_id: Some(turn_id.clone()),
2454            status,
2455            final_message,
2456            error: terminal_error,
2457            timestamp: OffsetDateTime::now_utc(),
2458        }))
2459        .await;
2460        Ok(())
2461    }
2462
2463    /// Stops accepting new turns, requests interruption for every active turn,
2464    /// and waits for their runtime tasks to reach terminal cleanup up to
2465    /// `timeout`. Repeated calls are safe and continue draining the same set.
2466    pub async fn drain_active_turns(&self, timeout: std::time::Duration) -> RuntimeDrainOutcome {
2467        let started_at = tokio::time::Instant::now();
2468        let persistence_failures_before =
2469            self.lifecycle_persistence_failures.load(Ordering::Acquire);
2470        let turn_admission = self.turn_admission.lock().await;
2471        self.accepting_turns.store(false, Ordering::Release);
2472        let active = self
2473            .active_turns
2474            .read()
2475            .await
2476            .iter()
2477            .map(|(turn_id, handle)| (turn_id.clone(), handle.thread_id.clone()))
2478            .collect::<Vec<_>>();
2479        let draining = self
2480            .turn_drains
2481            .read()
2482            .await
2483            .iter()
2484            .map(|(turn_id, drain)| (turn_id.clone(), drain.thread_id.clone()))
2485            .collect::<Vec<_>>();
2486        drop(turn_admission);
2487
2488        let mut interrupted_turns = std::collections::BTreeMap::new();
2489        for (turn_id, thread_id) in active.iter().chain(draining.iter()) {
2490            interrupted_turns.insert(turn_id.clone(), thread_id.clone());
2491        }
2492        let interrupted_turn_ids = interrupted_turns.keys().cloned().collect::<Vec<_>>();
2493        for (turn_id, thread_id) in active {
2494            let _ = self
2495                .interrupt_turn_with_reason(thread_id, turn_id, TurnLifecycleReason::Shutdown)
2496                .await;
2497        }
2498
2499        let wait_for_empty = async {
2500            loop {
2501                let notified = self.active_turns_changed.notified();
2502                if self.active_turns.read().await.is_empty()
2503                    && self.turn_drains.read().await.is_empty()
2504                {
2505                    return;
2506                }
2507                notified.await;
2508            }
2509        };
2510        let outcome = if tokio::time::timeout(timeout, wait_for_empty).await.is_ok() {
2511            if self.lifecycle_persistence_failures.load(Ordering::Acquire)
2512                > persistence_failures_before
2513            {
2514                RuntimeDrainOutcome::PersistenceFailed {
2515                    interrupted_turn_ids,
2516                    remaining_turn_ids: Vec::new(),
2517                }
2518            } else {
2519                RuntimeDrainOutcome::Clean {
2520                    interrupted_turn_ids,
2521                }
2522            }
2523        } else {
2524            let active_remaining = self
2525                .active_turns
2526                .read()
2527                .await
2528                .iter()
2529                .map(|(turn_id, handle)| (turn_id.clone(), handle.drain.clone()))
2530                .collect::<Vec<_>>();
2531            let draining_remaining = self
2532                .turn_drains
2533                .read()
2534                .await
2535                .iter()
2536                .map(|(turn_id, drain)| (turn_id.clone(), drain.clone()))
2537                .collect::<Vec<_>>();
2538            let remaining = active_remaining
2539                .into_iter()
2540                .chain(draining_remaining)
2541                .collect::<std::collections::BTreeMap<_, _>>();
2542            let remaining_turn_ids = remaining.keys().cloned().collect::<Vec<_>>();
2543            for turn_id in &remaining_turn_ids {
2544                let handle = remaining
2545                    .get(turn_id)
2546                    .expect("remaining turn ID must have a drain handle");
2547                self.record_turn_lifecycle(
2548                    handle.thread_id.clone(),
2549                    turn_id.clone(),
2550                    TurnLifecycleState::InterruptRequested,
2551                    TurnCleanupState::TimedOut,
2552                    Some(TurnLifecycleReason::Shutdown),
2553                )
2554                .await;
2555            }
2556            if self.lifecycle_persistence_failures.load(Ordering::Acquire)
2557                > persistence_failures_before
2558            {
2559                RuntimeDrainOutcome::PersistenceFailed {
2560                    interrupted_turn_ids,
2561                    remaining_turn_ids,
2562                }
2563            } else {
2564                RuntimeDrainOutcome::DeadlineExceeded {
2565                    interrupted_turn_ids,
2566                    remaining_turn_ids,
2567                }
2568            }
2569        };
2570        self.record_lifecycle_drain_outcome(started_at, &outcome);
2571        outcome
2572    }
2573
2574    /// Enables turn submission after a prior drain attempt. This is intended
2575    /// for embedders that keep a runtime alive after a bounded drain timeout.
2576    pub fn resume_accepting_turns(&self) {
2577        self.accepting_turns.store(true, Ordering::Release);
2578    }
2579
2580    pub async fn turn_lifecycle_snapshot(
2581        &self,
2582        thread_id: &ThreadId,
2583    ) -> anyhow::Result<TurnLifecycleSnapshot> {
2584        let Some(store) = &self.thread_store else {
2585            return Ok(TurnLifecycleSnapshot::default());
2586        };
2587        let extension_states = store.load_extension_states(thread_id).await?;
2588        Ok(turn_lifecycle_snapshot(&extension_states))
2589    }
2590
2591    pub async fn load_thread(
2592        &self,
2593        thread_id: &ThreadId,
2594    ) -> anyhow::Result<Option<ThreadSnapshot>> {
2595        let loaded = if let Some(store) = &self.thread_store {
2596            store.load_thread(thread_id).await?
2597        } else {
2598            None
2599        };
2600        if let Some(snapshot) = loaded.as_ref() {
2601            let live_turn_ids = self
2602                .active_turns
2603                .read()
2604                .await
2605                .keys()
2606                .cloned()
2607                .chain(self.turn_drains.read().await.keys().cloned())
2608                .collect::<std::collections::HashSet<_>>();
2609            let lifecycle = turn_lifecycle_snapshot(&snapshot.extension_states);
2610            for record in lifecycle.records.into_iter().filter(|record| {
2611                record.state.requires_recovery() && !live_turn_ids.contains(&record.turn_id)
2612            }) {
2613                self.lifecycle_restart_reconciliations
2614                    .fetch_add(1, Ordering::AcqRel);
2615                self.record_turn_lifecycle(
2616                    record.thread_id,
2617                    record.turn_id,
2618                    TurnLifecycleState::RecoveryNeeded,
2619                    TurnCleanupState::Unknown,
2620                    Some(TurnLifecycleReason::RuntimeRestart),
2621                )
2622                .await;
2623            }
2624            self.emit(RoderEvent::ThreadLoaded(ThreadLoaded {
2625                thread_id: thread_id.clone(),
2626                timestamp: OffsetDateTime::now_utc(),
2627            }))
2628            .await;
2629        }
2630        Ok(loaded)
2631    }
2632
2633    pub async fn workspace_for_thread(&self, thread_id: &ThreadId) -> anyhow::Result<String> {
2634        if let Some(store) = &self.thread_store {
2635            let snapshot = store
2636                .load_thread(thread_id)
2637                .await?
2638                .ok_or_else(|| anyhow::anyhow!("thread not found: {thread_id}"));
2639            match snapshot {
2640                Ok(snapshot) => {
2641                    if let Some(metadata) = snapshot.metadata {
2642                        // Fork-backed threads fail closed before any write
2643                        // when their workspace was removed out-of-band.
2644                        if let Some(fork) = &metadata.workspace_fork
2645                            && fork.status == roder_api::forks::ForkStatus::Active
2646                            && !std::path::Path::new(&metadata.workspace).is_dir()
2647                        {
2648                            anyhow::bail!(
2649                                "workspace fork {} is missing its workspace at {}; restore it or \
2650                                 remove the fork before running turns in this thread",
2651                                fork.id,
2652                                metadata.workspace
2653                            );
2654                        }
2655                        return Ok(metadata.workspace);
2656                    }
2657                    eprintln!(
2658                        "thread metadata missing while resolving workspace for {thread_id}; falling back to runtime workspace"
2659                    );
2660                }
2661                Err(err) => {
2662                    eprintln!(
2663                        "thread missing while resolving workspace for {thread_id}: {err}; falling back to runtime workspace"
2664                    );
2665                }
2666            }
2667        }
2668        Ok(self.workspace.display().to_string())
2669    }
2670
2671    pub(crate) async fn selection_mode_for_thread(
2672        &self,
2673        thread_id: &ThreadId,
2674    ) -> anyhow::Result<Option<ModelSelectionMode>> {
2675        let Some(store) = &self.thread_store else {
2676            return Ok(None);
2677        };
2678        Ok(store
2679            .load_thread(thread_id)
2680            .await?
2681            .and_then(|snapshot| snapshot.metadata)
2682            .and_then(|metadata| {
2683                metadata
2684                    .selection_mode
2685                    .or_else(|| match (metadata.provider, metadata.model) {
2686                        (Some(provider), Some(model)) => {
2687                            Some(ModelSelectionMode::manual(provider, model, None))
2688                        }
2689                        _ => None,
2690                    })
2691            }))
2692    }
2693
2694    /// Concrete provider/model for configured-role subagents (`task`,
2695    /// `agent_swarm`). Prefers the live parent turn selection so children stay
2696    /// on SuperGrok/Grok (or whatever the lead is using) instead of process
2697    /// startup defaults such as openai/gpt-5.5.
2698    pub(crate) async fn parent_model_selection_for_subagents(
2699        &self,
2700        thread_id: &ThreadId,
2701        turn_id: &TurnId,
2702    ) -> Option<roder_api::inference::ModelSelection> {
2703        if let Some(selection) = self.active_turn_selections.read().await.get(turn_id).cloned() {
2704            return Some(selection.concrete_selection());
2705        }
2706        if let Ok(Some(selection)) = self.selection_mode_for_thread(thread_id).await {
2707            return Some(selection.concrete_selection());
2708        }
2709        let cfg = self.status().await;
2710        if cfg.default_provider.trim().is_empty() || cfg.default_model.trim().is_empty() {
2711            return None;
2712        }
2713        Some(roder_api::inference::ModelSelection {
2714            provider: cfg.default_provider,
2715            model: cfg.default_model,
2716        })
2717    }
2718
2719    /// Per-thread tool allowlist, developer instructions, and external tools persisted at thread creation.
2720    pub(crate) async fn thread_turn_overrides(
2721        &self,
2722        thread_id: &ThreadId,
2723    ) -> anyhow::Result<ThreadTurnOverrides> {
2724        let Some(store) = &self.thread_store else {
2725            return Ok(ThreadTurnOverrides::default());
2726        };
2727        Ok(store
2728            .load_thread_metadata(thread_id)
2729            .await?
2730            .map(|metadata| ThreadTurnOverrides {
2731                tool_allowlist: metadata.tool_allowlist,
2732                developer_instructions: metadata.developer_instructions,
2733                external_tools: metadata.external_tools,
2734            })
2735            .unwrap_or_default())
2736    }
2737
2738    pub async fn set_thread_selection_mode(
2739        &self,
2740        thread_id: &ThreadId,
2741        selection_mode: ModelSelectionMode,
2742    ) -> anyhow::Result<()> {
2743        let Some(store) = &self.thread_store else {
2744            return Ok(());
2745        };
2746        let Some(snapshot) = store.load_thread(thread_id).await? else {
2747            anyhow::bail!("thread not found: {thread_id}");
2748        };
2749        let Some(mut metadata) = snapshot.metadata else {
2750            return Ok(());
2751        };
2752        let concrete = selection_mode.concrete_selection();
2753        metadata.provider = Some(concrete.provider);
2754        metadata.model = Some(concrete.model);
2755        metadata.selection_mode = Some(selection_mode);
2756        metadata.updated_at = OffsetDateTime::now_utc();
2757        store.update_thread_metadata(metadata).await?;
2758        Ok(())
2759    }
2760
2761    async fn runner_session_for_thread(
2762        &self,
2763        thread_id: &ThreadId,
2764    ) -> anyhow::Result<Option<(RunnerDestination, Arc<dyn RemoteRunnerSession>)>> {
2765        let metadata = if let Some(store) = &self.thread_store {
2766            store.load_thread_metadata(thread_id).await?
2767        } else {
2768            None
2769        };
2770        // An explicit per-thread binding wins over the runtime-level destination.
2771        let destination = metadata
2772            .as_ref()
2773            .and_then(|metadata| metadata.runner_binding.as_ref())
2774            .map(|binding| binding.destination.clone())
2775            .or(self.config.read().await.remote_runner_destination.clone());
2776        let Some(destination) = destination else {
2777            self.evict_runner_session(thread_id).await;
2778            return Ok(None);
2779        };
2780        let provider = self
2781            .registry
2782            .remote_runner_providers
2783            .iter()
2784            .find(|provider| provider.id() == destination.provider_id)
2785            .cloned()
2786            .ok_or_else(|| {
2787                anyhow::anyhow!(
2788                    "remote runner provider {:?} is not installed",
2789                    destination.provider_id
2790                )
2791            })?;
2792        let persisted_state = metadata.and_then(|metadata| metadata.runner_state);
2793        let slot = {
2794            let mut sessions = self.runner_sessions.lock().await;
2795            match sessions.get(thread_id) {
2796                Some(slot) if slot.matches(&destination) => slot.clone(),
2797                _ => {
2798                    let slot = Arc::new(RunnerSessionSlot::empty(&destination));
2799                    sessions.insert(thread_id.clone(), slot.clone());
2800                    slot
2801                }
2802            }
2803        };
2804
2805        loop {
2806            let initialized = slot.initialized.notified();
2807            let mut state = slot.state.lock().await;
2808            match &*state {
2809                RunnerSessionSlotState::Ready(cached) => {
2810                    return Ok(Some((cached.destination.clone(), cached.session.clone())));
2811                }
2812                RunnerSessionSlotState::Failed(error) => {
2813                    return Err(anyhow::anyhow!(error.to_string()));
2814                }
2815                RunnerSessionSlotState::Initializing => {
2816                    drop(state);
2817                    initialized.await;
2818                }
2819                RunnerSessionSlotState::Empty => {
2820                    *state = RunnerSessionSlotState::Initializing;
2821                    drop(state);
2822                    self.spawn_runner_session_initialization(
2823                        thread_id.clone(),
2824                        destination.clone(),
2825                        provider.clone(),
2826                        persisted_state.clone(),
2827                        slot.clone(),
2828                    );
2829                }
2830            }
2831        }
2832    }
2833
2834    fn spawn_runner_session_initialization(
2835        &self,
2836        thread_id: ThreadId,
2837        destination: RunnerDestination,
2838        provider: Arc<dyn RemoteRunnerProvider>,
2839        persisted_state: Option<RunnerSessionState>,
2840        slot: Arc<RunnerSessionSlot>,
2841    ) {
2842        let thread_store = self.thread_store.clone();
2843        let runner_sessions = self.runner_sessions.clone();
2844        // The task intentionally outlives the turn that first requested the
2845        // workspace. Interrupting that turn must not strand the slot in
2846        // `Initializing` and deadlock every later tool or lifecycle call.
2847        drop(tokio::spawn(async move {
2848            let initialized = async {
2849                let session = if let Some(state) = persisted_state
2850                    && state.provider_id == destination.provider_id
2851                    && state.destination_id == destination.id
2852                {
2853                    match provider.resume_session(state).await {
2854                        Ok(session) => session,
2855                        Err(_) => provider.create_session(destination.clone()).await?,
2856                    }
2857                } else {
2858                    provider.create_session(destination.clone()).await?
2859                };
2860                let resolved = (destination.clone(), session.clone());
2861                // Persist before releasing the singleflight or dispatching
2862                // the first tool. A later process can rejoin even if the turn
2863                // that requested initialization has already been interrupted.
2864                Self::persist_runner_state_in_store(
2865                    thread_store.as_ref(),
2866                    &thread_id,
2867                    Some(&resolved),
2868                )
2869                .await?;
2870                Ok::<_, anyhow::Error>(CachedRunnerSession {
2871                    destination,
2872                    session,
2873                })
2874            }
2875            .await;
2876
2877            match initialized {
2878                Ok(cached) => {
2879                    *slot.state.lock().await = RunnerSessionSlotState::Ready(cached);
2880                    slot.initialized.notify_waiters();
2881                }
2882                Err(error) => {
2883                    let message: Arc<str> = Arc::from(format!("{error:#}"));
2884                    *slot.state.lock().await = RunnerSessionSlotState::Failed(message);
2885                    slot.initialized.notify_waiters();
2886                    let mut sessions = runner_sessions.lock().await;
2887                    let is_current = sessions
2888                        .get(&thread_id)
2889                        .is_some_and(|current| Arc::ptr_eq(current, &slot));
2890                    if is_current {
2891                        sessions.remove(&thread_id);
2892                    }
2893                }
2894            }
2895        }));
2896    }
2897
2898    async fn evict_runner_session(&self, thread_id: &ThreadId) {
2899        self.runner_sessions.lock().await.remove(thread_id);
2900    }
2901
2902    async fn cache_runner_session(
2903        &self,
2904        thread_id: &ThreadId,
2905        destination: RunnerDestination,
2906        session: Arc<dyn RemoteRunnerSession>,
2907    ) {
2908        let cached = CachedRunnerSession {
2909            destination,
2910            session,
2911        };
2912        self.runner_sessions.lock().await.insert(
2913            thread_id.clone(),
2914            Arc::new(RunnerSessionSlot::ready(cached)),
2915        );
2916    }
2917
2918    /// Read a thread's explicit runner binding without creating or resuming a
2919    /// session. Tool routing uses this before local previews so runner-backed
2920    /// and fail-closed hosted calls never inspect the host workspace.
2921    pub(crate) async fn runner_binding_for_thread(
2922        &self,
2923        thread_id: &ThreadId,
2924    ) -> anyhow::Result<Option<ThreadRunnerBinding>> {
2925        let Some(store) = &self.thread_store else {
2926            return Ok(None);
2927        };
2928        Ok(store
2929            .load_thread_metadata(thread_id)
2930            .await?
2931            .and_then(|metadata| metadata.runner_binding))
2932    }
2933
2934    /// Resolve a previously-read binding into a live remote workspace. The
2935    /// session remains lazy: callers invoke this only after policy approval.
2936    pub(crate) async fn remote_workspace_for_binding(
2937        &self,
2938        thread_id: &ThreadId,
2939        binding: ThreadRunnerBinding,
2940    ) -> anyhow::Result<Arc<RemoteWorkspace>> {
2941        let session = self
2942            .runner_session_for_thread(thread_id)
2943            .await?
2944            .map(|(_, session)| session)
2945            .ok_or_else(|| {
2946                anyhow::anyhow!("runner-bound thread {thread_id} has no runner session")
2947            })?;
2948        Ok(Arc::new(RemoteWorkspace {
2949            session,
2950            root: binding.workspace,
2951            read_roots: binding.read_roots,
2952        }))
2953    }
2954
2955    pub(crate) async fn runner_tool_execution_lock(
2956        &self,
2957        session: &dyn RemoteRunnerSession,
2958    ) -> Arc<RunnerToolExecutionMutex> {
2959        let state = session.state();
2960        let key = (state.provider_id, state.session_id);
2961        let mut locks = self.runner_tool_execution_locks.lock().await;
2962        if let Some(lock) = locks.get(&key).and_then(Weak::upgrade) {
2963            return lock;
2964        }
2965
2966        locks.retain(|_, lock| lock.strong_count() > 0);
2967        let lock = Arc::new(Mutex::new(()));
2968        locks.insert(key, Arc::downgrade(&lock));
2969        lock
2970    }
2971
2972    async fn persist_runner_state(
2973        &self,
2974        thread_id: &ThreadId,
2975        runner: Option<&(RunnerDestination, Arc<dyn RemoteRunnerSession>)>,
2976    ) -> anyhow::Result<()> {
2977        Self::persist_runner_state_in_store(self.thread_store.as_ref(), thread_id, runner).await
2978    }
2979
2980    async fn persist_runner_state_in_store(
2981        store: Option<&Arc<dyn ThreadStore>>,
2982        thread_id: &ThreadId,
2983        runner: Option<&(RunnerDestination, Arc<dyn RemoteRunnerSession>)>,
2984    ) -> anyhow::Result<()> {
2985        let Some((destination, session)) = runner else {
2986            return Ok(());
2987        };
2988        let Some(store) = store else {
2989            return Ok(());
2990        };
2991        let Some(snapshot) = store.load_thread(thread_id).await? else {
2992            return Ok(());
2993        };
2994        let Some(mut metadata) = snapshot.metadata else {
2995            return Ok(());
2996        };
2997        metadata.runner_destination = Some(destination.clone());
2998        metadata.runner_state = Some(session.state());
2999        metadata.updated_at = OffsetDateTime::now_utc();
3000        store.update_thread_metadata(metadata).await?;
3001        Ok(())
3002    }
3003
3004    fn remote_runner_provider_by_id(
3005        &self,
3006        provider_id: &str,
3007    ) -> Option<Arc<dyn RemoteRunnerProvider>> {
3008        self.registry
3009            .remote_runner_providers
3010            .iter()
3011            .find(|provider| provider.id() == provider_id)
3012            .cloned()
3013    }
3014
3015    /// Resolve the live runner session for a thread along with its provider,
3016    /// failing clearly when the thread is not runner-bound.
3017    async fn thread_runner_session(
3018        &self,
3019        thread_id: &ThreadId,
3020    ) -> anyhow::Result<(
3021        RunnerDestination,
3022        Arc<dyn RemoteRunnerProvider>,
3023        Arc<dyn RemoteRunnerSession>,
3024    )> {
3025        let Some((destination, session)) = self.runner_session_for_thread(thread_id).await? else {
3026            anyhow::bail!("thread {thread_id} is not bound to a remote runner");
3027        };
3028        let provider = self
3029            .remote_runner_provider_by_id(&destination.provider_id)
3030            .ok_or_else(|| {
3031                anyhow::anyhow!(
3032                    "remote runner provider {:?} is not installed",
3033                    destination.provider_id
3034                )
3035            })?;
3036        Ok((destination, provider, session))
3037    }
3038
3039    /// Pause a runner-bound thread's session toward standby and persist the
3040    /// post-pause state. Errors if the provider is not pausable.
3041    pub async fn pause_thread_runner(
3042        &self,
3043        thread_id: &ThreadId,
3044    ) -> anyhow::Result<RunnerSessionState> {
3045        let (destination, provider, session) = self.thread_runner_session(thread_id).await?;
3046        anyhow::ensure!(
3047            provider.capabilities().pausable,
3048            "remote runner provider {:?} does not support pausing",
3049            destination.provider_id
3050        );
3051        let state = session.pause().await?;
3052        self.persist_runner_state(thread_id, Some(&(destination, session)))
3053            .await?;
3054        Ok(state)
3055    }
3056
3057    /// Resume (wake) a runner-bound thread's paused session and persist state.
3058    pub async fn resume_thread_runner(
3059        &self,
3060        thread_id: &ThreadId,
3061    ) -> anyhow::Result<RunnerSessionState> {
3062        let (destination, _provider, session) = self.thread_runner_session(thread_id).await?;
3063        let state = session.resume().await?;
3064        self.persist_runner_state(thread_id, Some(&(destination, session)))
3065            .await?;
3066        Ok(state)
3067    }
3068
3069    /// Detach a runner-bound thread's session: persist the durable, rejoinable
3070    /// state and leave the remote sandbox alive. Errors if not detachable.
3071    pub async fn detach_thread_runner(
3072        &self,
3073        thread_id: &ThreadId,
3074    ) -> anyhow::Result<RunnerSessionState> {
3075        let (destination, provider, session) = self.thread_runner_session(thread_id).await?;
3076        anyhow::ensure!(
3077            provider.capabilities().detachable,
3078            "remote runner provider {:?} does not support detaching",
3079            destination.provider_id
3080        );
3081        let state = session.detach().await?;
3082        // Persist the detached state explicitly so a later turn or process
3083        // rejoins the same sandbox instead of provisioning a new one.
3084        if let Some(store) = &self.thread_store
3085            && let Some(snapshot) = store.load_thread(thread_id).await?
3086            && let Some(mut metadata) = snapshot.metadata
3087        {
3088            metadata.runner_destination = Some(destination);
3089            metadata.runner_state = Some(state.clone());
3090            metadata.updated_at = OffsetDateTime::now_utc();
3091            store.update_thread_metadata(metadata).await?;
3092        }
3093        self.evict_runner_session(thread_id).await;
3094        Ok(state)
3095    }
3096
3097    /// Rejoin a previously created sandbox from a thread's persisted runner
3098    /// state without provisioning a new one. An optional `sandbox` overrides the
3099    /// persisted sandbox name (recovery by name). Persists refreshed state.
3100    pub async fn rejoin_thread_runner(
3101        &self,
3102        thread_id: &ThreadId,
3103        sandbox: Option<String>,
3104    ) -> anyhow::Result<RunnerSessionState> {
3105        let store = self
3106            .thread_store
3107            .as_ref()
3108            .ok_or_else(|| anyhow::anyhow!("thread store is required to rejoin a runner"))?;
3109        let metadata = store
3110            .load_thread_metadata(thread_id)
3111            .await?
3112            .ok_or_else(|| anyhow::anyhow!("thread {thread_id} has no metadata"))?;
3113        let destination = metadata
3114            .runner_binding
3115            .as_ref()
3116            .map(|binding| binding.destination.clone())
3117            .or_else(|| metadata.runner_destination.clone())
3118            .ok_or_else(|| anyhow::anyhow!("thread {thread_id} is not bound to a remote runner"))?;
3119        let mut state = metadata
3120            .runner_state
3121            .clone()
3122            .ok_or_else(|| anyhow::anyhow!("thread {thread_id} has no persisted runner state"))?;
3123        if let Some(sandbox) = sandbox
3124            && let Some(object) = state.metadata.as_object_mut()
3125        {
3126            object.insert("sandbox_name".to_string(), serde_json::Value::from(sandbox));
3127        }
3128        let provider = self
3129            .remote_runner_provider_by_id(&destination.provider_id)
3130            .ok_or_else(|| {
3131                anyhow::anyhow!(
3132                    "remote runner provider {:?} is not installed",
3133                    destination.provider_id
3134                )
3135            })?;
3136        let session = provider.rejoin_session(state).await?;
3137        self.persist_runner_state(thread_id, Some(&(destination.clone(), session.clone())))
3138            .await?;
3139        self.cache_runner_session(thread_id, destination, session.clone())
3140            .await;
3141        Ok(session.state())
3142    }
3143
3144    async fn record_thread_usage_metadata(
3145        &self,
3146        thread_id: &ThreadId,
3147        usage: &TokenUsage,
3148    ) -> anyhow::Result<()> {
3149        if usage.is_empty() {
3150            return Ok(());
3151        }
3152        let Some(store) = &self.thread_store else {
3153            return Ok(());
3154        };
3155        let Some(snapshot) = store.load_thread(thread_id).await? else {
3156            return Ok(());
3157        };
3158        let Some(mut metadata) = snapshot.metadata else {
3159            return Ok(());
3160        };
3161        metadata
3162            .usage
3163            .get_or_insert_with(ThreadUsageMetadata::default)
3164            .add_token_usage(usage);
3165        metadata.updated_at = OffsetDateTime::now_utc();
3166        store.update_thread_metadata(metadata).await?;
3167        Ok(())
3168    }
3169
3170    pub fn start_turn(
3171        self: &Arc<Self>,
3172        mut req: StartTurnRequest,
3173    ) -> BoxFuture<'_, anyhow::Result<TurnId>> {
3174        Box::pin(async move {
3175            let turn_admission = self.turn_admission.lock().await;
3176            anyhow::ensure!(
3177                self.accepting_turns.load(Ordering::Acquire),
3178                "runtime is quiescing and cannot accept new turns"
3179            );
3180            req.workspace = validate_thread_workspace(&req.workspace)?;
3181            let team_member = self.teams.member_for_thread(&req.thread_id).await;
3182            let cfg = self.config.read().await.clone();
3183            let provider = req
3184                .provider_override
3185                .clone()
3186                .unwrap_or_else(|| cfg.default_provider.clone());
3187            self.engine_for(&provider)?;
3188            let turn_id = uuid::Uuid::new_v4().to_string();
3189            let mut initial_mailbox_ack = None;
3190            if let Some((team_id, member)) = &team_member {
3191                let pending = self
3192                    .teams
3193                    .reserve_pending_mailbox_messages(team_id, &member.id, &turn_id)
3194                    .await?;
3195                if !pending.is_empty()
3196                    && let Some(team) = self.read_team(team_id).await
3197                {
3198                    let mailbox = format_mailbox_messages(&team, &pending);
3199                    req.message = if req.message.trim().is_empty() {
3200                        mailbox
3201                    } else {
3202                        format!("{mailbox}\n\n[Direct task input]\n{}", req.message)
3203                    };
3204                    initial_mailbox_ack = Some(MailboxDeliveryAck {
3205                        team_id: team_id.clone(),
3206                        message_ids: pending.iter().map(|message| message.id.clone()).collect(),
3207                    });
3208                }
3209            }
3210            let (abort_handle, abort_registration) = AbortHandle::new_pair();
3211            let drain = Arc::new(TurnDrainHandle {
3212                thread_id: req.thread_id.clone(),
3213                interrupt_requested: AtomicBool::new(false),
3214                interrupt_reason: Mutex::new(None),
3215                completed: AtomicBool::new(false),
3216                completed_notify: Notify::new(),
3217            });
3218            let active = ActiveTurnHandle {
3219                thread_id: req.thread_id.clone(),
3220                abort: abort_handle,
3221                steers: Arc::new(Mutex::new(Vec::new())),
3222                drain,
3223            };
3224            self.active_turns
3225                .write()
3226                .await
3227                .insert(turn_id.clone(), active);
3228            self.record_turn_lifecycle(
3229                req.thread_id.clone(),
3230                turn_id.clone(),
3231                TurnLifecycleState::Running,
3232                TurnCleanupState::NotRequested,
3233                None,
3234            )
3235            .await;
3236            self.active_turn_contexts.write().await.insert(
3237                turn_id.clone(),
3238                InheritedTurnContext {
3239                    workspace: req.workspace.clone(),
3240                    instructions: req.instructions.clone(),
3241                    developer_context: req.developer_context.clone(),
3242                },
3243            );
3244            if let Some((team_id, member)) = team_member {
3245                let updated = match self
3246                    .teams
3247                    .update_member(&team_id, &member.id, |member| {
3248                        member.current_turn_id = Some(turn_id.clone());
3249                        member.status = TeamMemberStatus::Running;
3250                        member.final_message = None;
3251                        member.terminal_error = None;
3252                    })
3253                    .await
3254                {
3255                    Ok(updated) => updated,
3256                    Err(error) => {
3257                        self.active_turns.write().await.remove(&turn_id);
3258                        self.active_turn_contexts.write().await.remove(&turn_id);
3259                        self.teams
3260                            .release_mailbox_reservations_for_turn(&turn_id)
3261                            .await;
3262                        return Err(error);
3263                    }
3264                };
3265                if let Some(member) = updated
3266                    .members
3267                    .into_iter()
3268                    .find(|candidate| candidate.id == member.id)
3269                {
3270                    self.emit(RoderEvent::TeamMemberStatusChanged(
3271                        TeamMemberStatusChanged {
3272                            team_id,
3273                            member_id: member.id,
3274                            member_thread_id: member.thread_id,
3275                            status: TeamMemberStatus::Running,
3276                            timestamp: OffsetDateTime::now_utc(),
3277                        },
3278                    ))
3279                    .await;
3280                }
3281            }
3282            let runtime = Arc::clone(self);
3283            let turn_req = req;
3284            let thread_id_for_task = turn_req.thread_id.clone();
3285            let turn_id_for_task = turn_id.clone();
3286            tokio::spawn(async move {
3287                let result = Abortable::new(
3288                    runtime.run_turn(turn_req, turn_id_for_task.clone(), initial_mailbox_ack),
3289                    abort_registration,
3290                )
3291                .await;
3292                /*
3293                 * A failed sibling in a parallel tool batch drops in-flight external tool
3294                 * futures (`try_join_all` in `route_tool_calls`), stranding their
3295                 * `pending_external_tool_calls` entries. Sweep before reporting the turn
3296                 * outcome so every `thread/toolExecutionRequested` gets a terminal
3297                 * resolution; on clean completion the map holds nothing for this turn.
3298                 */
3299                runtime
3300                    .cancel_pending_external_tool_calls_for_turn(&turn_id_for_task)
3301                    .await;
3302                let completed = matches!(&result, Ok(Ok(TurnRunOutcome::Completed)));
3303                match &result {
3304                    Ok(Err(err)) => {
3305                        let (cleanup, ownership) =
3306                            runtime.await_provider_turn_cleanup(&turn_id_for_task).await;
3307                        // run_turn emits failures after the stream starts; this covers setup/startup errors.
3308                        runtime
3309                            .emit(RoderEvent::TurnFailed(TurnFailed {
3310                                thread_id: thread_id_for_task.clone(),
3311                                turn_id: turn_id_for_task.clone(),
3312                                error: err.to_string(),
3313                                error_kind: None,
3314                                usage: None,
3315                                timestamp: OffsetDateTime::now_utc(),
3316                            }))
3317                            .await;
3318                        runtime
3319                            .record_turn_lifecycle_with_ownership(
3320                                thread_id_for_task.clone(),
3321                                turn_id_for_task.clone(),
3322                                TurnLifecycleState::Failed,
3323                                cleanup,
3324                                Some(TurnLifecycleReason::ProviderFailure),
3325                                ownership,
3326                            )
3327                            .await;
3328                        let _ = runtime
3329                            .complete_team_member_turn_with_result(
3330                                &thread_id_for_task,
3331                                &turn_id_for_task,
3332                                TeamMemberStatus::Failed,
3333                                None,
3334                                Some(err.to_string()),
3335                            )
3336                            .await;
3337                    }
3338                    Ok(Ok(TurnRunOutcome::Stopped)) => {
3339                        let (cleanup, ownership) =
3340                            runtime.await_provider_turn_cleanup(&turn_id_for_task).await;
3341                        runtime
3342                            .record_turn_lifecycle_with_ownership(
3343                                thread_id_for_task.clone(),
3344                                turn_id_for_task.clone(),
3345                                TurnLifecycleState::Failed,
3346                                cleanup,
3347                                Some(TurnLifecycleReason::ProviderFailure),
3348                                ownership,
3349                            )
3350                            .await;
3351                        let _ = runtime
3352                            .complete_team_member_turn_with_result(
3353                                &thread_id_for_task,
3354                                &turn_id_for_task,
3355                                TeamMemberStatus::Failed,
3356                                None,
3357                                Some("turn stopped before completion".to_string()),
3358                            )
3359                            .await;
3360                    }
3361                    Err(_) => {
3362                        let reason = if let Some(handle) = runtime
3363                            .turn_drains
3364                            .read()
3365                            .await
3366                            .get(&turn_id_for_task)
3367                            .cloned()
3368                        {
3369                            handle
3370                                .interrupt_reason
3371                                .lock()
3372                                .await
3373                                .unwrap_or(TurnLifecycleReason::RuntimeFailure)
3374                        } else {
3375                            TurnLifecycleReason::RuntimeFailure
3376                        };
3377                        let (cleanup, ownership) =
3378                            runtime.await_provider_turn_cleanup(&turn_id_for_task).await;
3379                        runtime
3380                            .record_turn_lifecycle_with_ownership(
3381                                thread_id_for_task.clone(),
3382                                turn_id_for_task.clone(),
3383                                TurnLifecycleState::Interrupted,
3384                                cleanup,
3385                                Some(reason),
3386                                ownership,
3387                            )
3388                            .await;
3389                        runtime
3390                            .emit(RoderEvent::TurnInterrupted(TurnInterrupted {
3391                                thread_id: thread_id_for_task.clone(),
3392                                turn_id: turn_id_for_task.clone(),
3393                                timestamp: OffsetDateTime::now_utc(),
3394                            }))
3395                            .await;
3396                        let _ = runtime
3397                            .complete_team_member_turn_with_result(
3398                                &thread_id_for_task,
3399                                &turn_id_for_task,
3400                                TeamMemberStatus::Interrupted,
3401                                None,
3402                                None,
3403                            )
3404                            .await;
3405                    }
3406                    Ok(Ok(TurnRunOutcome::Completed)) => {}
3407                }
3408                runtime
3409                    .teams
3410                    .release_mailbox_reservations_for_turn(&turn_id_for_task)
3411                    .await;
3412                runtime.active_turns.write().await.remove(&turn_id_for_task);
3413                if let Some(drain) = runtime.turn_drains.write().await.remove(&turn_id_for_task) {
3414                    drain.completed.store(true, Ordering::Release);
3415                    drain.completed_notify.notify_waiters();
3416                }
3417                runtime
3418                    .active_turn_selections
3419                    .write()
3420                    .await
3421                    .remove(&turn_id_for_task);
3422                runtime
3423                    .active_turn_contexts
3424                    .write()
3425                    .await
3426                    .remove(&turn_id_for_task);
3427                if !completed {
3428                    // Completion paths above consume registered provider cleanup
3429                    // handles. A setup failure before an engine can stream has no
3430                    // such handle; this is a harmless final defensive sweep.
3431                    let _ = runtime.provider_turn_cleanups.lock().map(|mut cleanups| {
3432                        cleanups.remove(&turn_id_for_task);
3433                    });
3434                }
3435                runtime.active_turns_changed.notify_waiters();
3436                if completed {
3437                    let _ = runtime
3438                        .continue_active_goal_after_turn(thread_id_for_task)
3439                        .await;
3440                }
3441            });
3442            drop(turn_admission);
3443            Ok(turn_id)
3444        })
3445    }
3446
3447    pub(crate) async fn has_active_turn_for_thread(&self, thread_id: &ThreadId) -> bool {
3448        self.active_turns
3449            .read()
3450            .await
3451            .values()
3452            .any(|handle| &handle.thread_id == thread_id)
3453    }
3454
3455    async fn ensure_codex_v2_team_capacity(
3456        &self,
3457        team: &TeamState,
3458        resuming_member_id: &str,
3459        candidate_model: Option<&str>,
3460    ) -> anyhow::Result<()> {
3461        if !is_codex_v2_team(team)
3462            && !candidate_model
3463                .is_some_and(|model| model_supports_reasoning_effort(model, REASONING_ULTRA))
3464        {
3465            return Ok(());
3466        }
3467        let active_thread_ids = self
3468            .active_turns
3469            .read()
3470            .await
3471            .values()
3472            .map(|handle| handle.thread_id.clone())
3473            .collect::<std::collections::HashSet<_>>();
3474        let resident_threads = 1 + team
3475            .members
3476            .iter()
3477            .filter(|member| {
3478                member.role != roder_api::teams::TeamMemberRole::Lead
3479                    && member.id != resuming_member_id
3480                    && active_thread_ids.contains(&member.thread_id)
3481            })
3482            .count();
3483        anyhow::ensure!(
3484            resident_threads < codex_v2::CODEX_V2_MAX_RESIDENT_TEAM_THREADS,
3485            "agent thread limit reached for this Codex V2 team: maximum {} resident threads (the lead plus 3 running subagents)",
3486            codex_v2::CODEX_V2_MAX_RESIDENT_TEAM_THREADS
3487        );
3488        Ok(())
3489    }
3490
3491    /// Number of currently running turns across all threads. Hosted runtime
3492    /// pools use this to avoid evicting tenants with active work.
3493    pub async fn active_turn_count(&self) -> usize {
3494        self.active_turns.read().await.len()
3495    }
3496
3497    pub async fn active_turn_for_thread(&self, thread_id: &ThreadId) -> Option<TurnId> {
3498        self.active_turns
3499            .read()
3500            .await
3501            .iter()
3502            .find_map(|(turn_id, handle)| (&handle.thread_id == thread_id).then(|| turn_id.clone()))
3503    }
3504
3505    pub async fn thread_activity(&self, thread_id: &ThreadId) -> ThreadActivity {
3506        let active_turn_id = self.active_turn_for_thread(thread_id).await;
3507        let draining_turn_id = if active_turn_id.is_none() {
3508            self.turn_drains
3509                .read()
3510                .await
3511                .iter()
3512                .find_map(|(turn_id, drain)| {
3513                    (&drain.thread_id == thread_id).then(|| turn_id.clone())
3514                })
3515        } else {
3516            None
3517        };
3518        let Some(active_turn_id) = active_turn_id.or(draining_turn_id) else {
3519            return ThreadActivity::default();
3520        };
3521
3522        let mut active_flags = Vec::new();
3523        {
3524            let pending_approvals = self.pending_tool_approvals.lock().await;
3525            if pending_approvals
3526                .values()
3527                .any(|pending| &pending.thread_id == thread_id && pending.turn_id == active_turn_id)
3528            {
3529                active_flags.push("approvalRequired".to_string());
3530            }
3531        }
3532        {
3533            let pending_inputs = self.pending_user_inputs.lock().await;
3534            if pending_inputs
3535                .values()
3536                .any(|pending| &pending.thread_id == thread_id && pending.turn_id == active_turn_id)
3537            {
3538                active_flags.push("userInputRequired".to_string());
3539            }
3540        }
3541        {
3542            let pending_external = self.pending_external_tool_calls.lock().await;
3543            if pending_external
3544                .values()
3545                .any(|pending| &pending.thread_id == thread_id && pending.turn_id == active_turn_id)
3546            {
3547                active_flags.push("externalToolPending".to_string());
3548            }
3549        }
3550        let interrupting = self
3551            .active_turns
3552            .read()
3553            .await
3554            .get(&active_turn_id)
3555            .is_some_and(|handle| handle.drain.interrupt_requested.load(Ordering::Acquire))
3556            || self
3557                .turn_drains
3558                .read()
3559                .await
3560                .get(&active_turn_id)
3561                .is_some_and(|drain| drain.interrupt_requested.load(Ordering::Acquire));
3562        if interrupting {
3563            active_flags.push("interrupting".to_string());
3564        }
3565        if self.pending_plan_exit().await.is_some_and(|pending| {
3566            &pending.thread_id == thread_id && pending.turn_id == active_turn_id
3567        }) {
3568            active_flags.push("planExitRequired".to_string());
3569        }
3570
3571        ThreadActivity {
3572            active_turn_id: Some(active_turn_id),
3573            active_flags,
3574        }
3575    }
3576
3577    pub async fn interrupt_turn(&self, thread_id: ThreadId, turn_id: TurnId) -> anyhow::Result<()> {
3578        self.interrupt_turn_with_reason(thread_id, turn_id, TurnLifecycleReason::UserInterrupt)
3579            .await
3580    }
3581
3582    async fn interrupt_turn_with_reason(
3583        &self,
3584        thread_id: ThreadId,
3585        turn_id: TurnId,
3586        reason: TurnLifecycleReason,
3587    ) -> anyhow::Result<()> {
3588        let handle = self.active_turns.write().await.remove(&turn_id);
3589        if let Some(handle) = handle {
3590            if handle
3591                .drain
3592                .interrupt_requested
3593                .swap(true, Ordering::AcqRel)
3594            {
3595                return Ok(());
3596            }
3597            *handle.drain.interrupt_reason.lock().await = Some(reason);
3598            self.turn_drains
3599                .write()
3600                .await
3601                .insert(turn_id.clone(), handle.drain.clone());
3602            let ownership = self.provider_cleanup_ownership(&turn_id);
3603            self.record_turn_lifecycle_with_ownership(
3604                thread_id.clone(),
3605                turn_id.clone(),
3606                TurnLifecycleState::InterruptRequested,
3607                TurnCleanupState::Requested,
3608                Some(reason),
3609                ownership,
3610            )
3611            .await;
3612            handle.abort.abort();
3613            self.active_turns_changed.notify_waiters();
3614        }
3615        self.active_turn_selections.write().await.remove(&turn_id);
3616        self.cancel_pending_external_tool_calls_for_turn(&turn_id)
3617            .await;
3618        Ok(())
3619    }
3620
3621    pub async fn steer_turn(
3622        &self,
3623        thread_id: ThreadId,
3624        turn_id: TurnId,
3625        message: String,
3626        images: Vec<InputImage>,
3627    ) -> anyhow::Result<()> {
3628        self.enqueue_turn_steer(thread_id, turn_id, message, images, None)
3629            .await
3630    }
3631
3632    pub(crate) async fn has_pending_turn_steers(&self, turn_id: &TurnId) -> bool {
3633        let active = self.active_turns.read().await.get(turn_id).cloned();
3634        let Some(active) = active else {
3635            return false;
3636        };
3637        let has_pending = !active.steers.lock().await.is_empty();
3638        has_pending
3639    }
3640
3641    async fn steer_turn_with_mailbox_ack(
3642        &self,
3643        thread_id: ThreadId,
3644        turn_id: TurnId,
3645        message: String,
3646        message_ids: Vec<String>,
3647        team_id: TeamId,
3648    ) -> anyhow::Result<()> {
3649        self.enqueue_turn_steer(
3650            thread_id,
3651            turn_id,
3652            message,
3653            Vec::new(),
3654            Some(MailboxDeliveryAck {
3655                team_id,
3656                message_ids,
3657            }),
3658        )
3659        .await
3660    }
3661
3662    async fn deliver_pending_team_mailbox(
3663        &self,
3664        team_id: &str,
3665        member_id: &str,
3666        member_thread_id: ThreadId,
3667        turn_id: TurnId,
3668    ) -> anyhow::Result<()> {
3669        let pending = self
3670            .teams
3671            .reserve_pending_mailbox_messages(team_id, member_id, &turn_id)
3672            .await?;
3673        if pending.is_empty() {
3674            return Ok(());
3675        }
3676        let message_ids = pending
3677            .iter()
3678            .map(|message| message.id.clone())
3679            .collect::<Vec<_>>();
3680        let team = self
3681            .read_team(team_id)
3682            .await
3683            .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
3684        if let Err(error) = self
3685            .steer_turn_with_mailbox_ack(
3686                member_thread_id,
3687                turn_id.clone(),
3688                format_mailbox_messages(&team, &pending),
3689                message_ids.clone(),
3690                team_id.to_string(),
3691            )
3692            .await
3693        {
3694            self.teams
3695                .release_mailbox_reservations(&turn_id, &message_ids)
3696                .await;
3697            return Err(error);
3698        }
3699        Ok(())
3700    }
3701
3702    async fn enqueue_turn_steer(
3703        &self,
3704        thread_id: ThreadId,
3705        turn_id: TurnId,
3706        message: String,
3707        images: Vec<InputImage>,
3708        mailbox_ack: Option<MailboxDeliveryAck>,
3709    ) -> anyhow::Result<()> {
3710        let message = message.trim().to_string();
3711        if message.is_empty() && images.is_empty() {
3712            return Ok(());
3713        }
3714
3715        let Some(active) = self.active_turns.read().await.get(&turn_id).cloned() else {
3716            anyhow::bail!("no active turn to steer");
3717        };
3718        active.steers.lock().await.push(QueuedTurnSteer {
3719            message: UserMessage::with_images(message.clone(), images),
3720            mailbox_ack,
3721        });
3722        self.emit(RoderEvent::TurnSteered(TurnSteered {
3723            thread_id,
3724            turn_id,
3725            message,
3726            timestamp: OffsetDateTime::now_utc(),
3727        }))
3728        .await;
3729        Ok(())
3730    }
3731
3732    pub async fn tool_specs(&self) -> Vec<roder_api::tools::ToolSpec> {
3733        let cfg = self.config.read().await;
3734        let model_profile =
3735            model_profile_for_provider_model(&cfg, &cfg.default_provider, &cfg.default_model);
3736        self.filtered_tool_specs(&cfg, &cfg.default_model, model_profile.as_ref(), &[], &[])
3737    }
3738
3739    pub fn subagent_definitions(&self) -> Vec<SubagentDefinition> {
3740        self.registry
3741            .subagent_dispatchers
3742            .iter()
3743            .flat_map(|dispatcher| dispatcher.definitions())
3744            .collect()
3745    }
3746
3747    async fn run_turn(
3748        self: &Arc<Self>,
3749        req: StartTurnRequest,
3750        turn_id: TurnId,
3751        initial_mailbox_ack: Option<MailboxDeliveryAck>,
3752    ) -> anyhow::Result<TurnRunOutcome> {
3753        let turn_started_at = OffsetDateTime::now_utc();
3754        self.emit(RoderEvent::TurnStarted(TurnStarted {
3755            thread_id: req.thread_id.clone(),
3756            turn_id: turn_id.clone(),
3757            runtime_profile: self.config.read().await.runtime_profile,
3758            timestamp: turn_started_at,
3759        }))
3760        .await;
3761        self.persist_turn_item(
3762            &req.thread_id,
3763            &turn_id,
3764            &TranscriptItem::UserMessage(UserMessage::with_images(
3765                req.message.clone(),
3766                req.images.clone(),
3767            )),
3768        )
3769        .await?;
3770        if let Some(ack) = initial_mailbox_ack {
3771            self.teams
3772                .mark_mailbox_messages_delivered(&ack.team_id, &turn_id, &ack.message_ids)
3773                .await?;
3774        }
3775
3776        let mut cfg = self.config.read().await.clone();
3777        let runtime_profile = cfg.runtime_profile;
3778        let turn_deadline = turn_deadline_for_config(&cfg);
3779        let deadline_finalization_reserve =
3780            crate::deadline_policy::finalization_reserve_seconds(cfg.turn_deadline_seconds);
3781        let selection_mode = self.selection_mode_for_thread(&req.thread_id).await?;
3782        let concrete_selection = selection_mode
3783            .as_ref()
3784            .map(ModelSelectionMode::concrete_selection);
3785        let default_provider = req
3786            .provider_override
3787            .clone()
3788            .or_else(|| {
3789                concrete_selection
3790                    .as_ref()
3791                    .map(|selection| selection.provider.clone())
3792            })
3793            .unwrap_or(cfg.default_provider.clone());
3794        let default_model = req
3795            .model_override
3796            .clone()
3797            .or_else(|| {
3798                concrete_selection
3799                    .as_ref()
3800                    .map(|selection| selection.model.clone())
3801            })
3802            .unwrap_or(cfg.default_model.clone());
3803        if let Some(reasoning) = req.reasoning_override.as_deref().or_else(|| {
3804            selection_mode
3805                .as_ref()
3806                .and_then(ModelSelectionMode::reasoning)
3807        }) {
3808            validate_reasoning_effort(&default_model, reasoning)?;
3809            cfg.reasoning = Some(reasoning.to_string());
3810        }
3811        let turn_has_concrete_model_override =
3812            req.provider_override.is_some() || req.model_override.is_some();
3813        let (turn_inference_router, turn_inference_router_profile) = match &selection_mode {
3814            Some(ModelSelectionMode::Auto {
3815                router_id, profile, ..
3816            }) if !turn_has_concrete_model_override => (
3817                RuntimeInferenceRouterConfig {
3818                    enabled: true,
3819                    router_id: Some(router_id.clone()),
3820                },
3821                profile.clone(),
3822            ),
3823            _ => (RuntimeInferenceRouterConfig::disabled(), None),
3824        };
3825        let mut provider = default_provider.clone();
3826        let mut model = default_model.clone();
3827        let mut model_profile = model_profile_for_provider_model(&cfg, &provider, &model);
3828        let workspace = req.workspace.clone();
3829        let mut transcript = self.transcript_for_turn(&req, &turn_id, &model).await?;
3830        let mut compacted_this_turn = transcript
3831            .iter()
3832            .any(crate::compaction::is_compaction_boundary);
3833        let effective_policy_mode = self.effective_policy_mode_for_thread(&req.thread_id).await;
3834        let agent_swarm_mode_active = self
3835            .effective_agent_swarm_mode_for_thread(&req.thread_id)
3836            .await;
3837        let ultra_mode_active = self.effective_ultra_mode_for_thread(&req.thread_id).await;
3838        let thread_overrides = self.thread_turn_overrides(&req.thread_id).await?;
3839        let mut final_assistant_text = String::new();
3840        let mut final_phase_messages = Vec::<AssistantMessage>::new();
3841        let mut final_reasoning_text = String::new();
3842        let mut final_provider_metadata = None;
3843        let mut exhausted_tool_rounds = true;
3844        let mut verification_gate =
3845            VerificationGateState::new(req.message.clone(), runtime_profile);
3846        let mut speed_policy = SpeedPolicyState::default();
3847        let mut reliability = TurnReliabilityState::default();
3848        let mut turn_usage = TokenUsage::default();
3849        // Overwritten on every inference step's Completed event so only the
3850        // terminal step's stop reason survives (mid-turn tool_use steps must
3851        // not leak onto turn/completed).
3852        let mut turn_finish_reason: Option<String> = None;
3853        let mut deadline_finalization_requested = false;
3854        let mut deadline_scoreable_completion_requested = false;
3855        let mut task_ledger_completion_reminders = 0_u8;
3856        let mut task_ledger_scoreable_checkpoints = 0_u8;
3857        let mut empty_tool_call_nudges_used = 0_u32;
3858        let mut provider_stream_retry_attempts = 0_u32;
3859        let mut context_limit_recovery_attempts = 0_u32;
3860        let mut routing_candidates = None;
3861        let routing_transcript_start = transcript.len().saturating_sub(1);
3862        let mut routing_escalations = 0_u32;
3863        let mut model_switch_summary_selection = None::<ModelSelection>;
3864
3865        'tool_rounds: for round_index in 0..MAX_TOOL_ROUNDS_PER_TURN {
3866            if let Some(deadline) = turn_deadline
3867                && deadline_expired(deadline)
3868            {
3869                self.fail_turn_due_to_deadline(&req.thread_id, &turn_id, deadline, &transcript)
3870                    .await?;
3871                return Ok(TurnRunOutcome::Stopped);
3872            }
3873            let steers = self.drain_turn_steers(&turn_id).await;
3874            self.append_steers(&req, &turn_id, &mut transcript, steers)
3875                .await?;
3876            if runtime_profile == RuntimeProfile::Eval
3877                && let Some(remaining) = crate::deadline_policy::should_start_finalization(
3878                    turn_deadline,
3879                    deadline_finalization_reserve,
3880                    deadline_finalization_requested || deadline_scoreable_completion_requested,
3881                )
3882            {
3883                if req.task_ledger_required
3884                    && task_ledger_completion_reminders < TASK_LEDGER_COMPLETION_REMINDER_LIMIT
3885                    && let Some(prompt) = task_ledger_completion_prompt(&transcript)
3886                {
3887                    task_ledger_completion_reminders += 1;
3888                    deadline_scoreable_completion_requested = true;
3889                    let item = TranscriptItem::UserMessage(UserMessage::text(
3890                        task_ledger_deadline_completion_prompt(
3891                            remaining,
3892                            deadline_finalization_reserve,
3893                            &prompt,
3894                        ),
3895                    ));
3896                    self.persist_turn_item(&req.thread_id, &turn_id, &item)
3897                        .await?;
3898                    transcript.push(item);
3899                    continue 'tool_rounds;
3900                } else {
3901                    self.start_deadline_finalization(
3902                        &req.thread_id,
3903                        &turn_id,
3904                        &mut transcript,
3905                        remaining,
3906                    )
3907                    .await?;
3908                    deadline_finalization_requested = true;
3909                }
3910            }
3911            if runtime_profile == RuntimeProfile::Eval
3912                && req.task_ledger_required
3913                && !deadline_finalization_requested
3914                && task_ledger_scoreable_checkpoints < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
3915                && let Some(remaining) = deadline_remaining_seconds(turn_deadline)
3916                && remaining <= TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS
3917                && remaining > deadline_finalization_reserve
3918                && let Some(prompt) = task_ledger_completion_prompt(&transcript)
3919            {
3920                task_ledger_scoreable_checkpoints += 1;
3921                let item = TranscriptItem::UserMessage(UserMessage::text(
3922                    task_ledger_scoreable_checkpoint_prompt(remaining, &prompt),
3923                ));
3924                self.persist_turn_item(&req.thread_id, &turn_id, &item)
3925                    .await?;
3926                transcript.push(item);
3927                continue 'tool_rounds;
3928            }
3929            if turn_inference_router.is_active() && routing_candidates.is_none() {
3930                routing_candidates =
3931                    Some(collect_inference_routing_candidates(&self.registry).await);
3932            }
3933            let routing_tools_model = model.clone();
3934            let routing_tools = self.filtered_tool_specs(
3935                &cfg,
3936                &model,
3937                model_profile.as_ref(),
3938                &thread_overrides.tool_allowlist,
3939                &thread_overrides.external_tools,
3940            );
3941            let prior_failures =
3942                transcript_failure_count_since(&transcript, routing_transcript_start)
3943                    .max(reliability.tool_failure_count())
3944                    .saturating_add(provider_stream_retry_attempts);
3945            let routing_selection = route_inference_selection(
3946                &self.registry,
3947                &turn_inference_router,
3948                InferenceRoutingRequest {
3949                    thread_id: &req.thread_id,
3950                    turn_id: &turn_id,
3951                    round_index: round_index as u32,
3952                    runtime_profile,
3953                    phase: speed_policy.phase(),
3954                    profile: turn_inference_router_profile.as_deref(),
3955                    default_selection: ModelSelection {
3956                        provider: default_provider.clone(),
3957                        model: default_model.clone(),
3958                    },
3959                    transcript: &transcript,
3960                    tools: &routing_tools,
3961                    candidates: routing_candidates.as_deref(),
3962                    prior_failures,
3963                    prior_escalations: routing_escalations,
3964                },
3965            )
3966            .await;
3967            if let Some(decision) = routing_selection.decision.clone() {
3968                if matches!(decision.outcome, InferenceRoutingOutcome::Escalated) {
3969                    routing_escalations = routing_escalations.saturating_add(1);
3970                }
3971                self.emit(RoderEvent::InferenceRoutingDecision(
3972                    InferenceRoutingDecisionEvent {
3973                        thread_id: req.thread_id.clone(),
3974                        turn_id: turn_id.clone(),
3975                        round_index: round_index as u32,
3976                        default_selection: ModelSelection {
3977                            provider: default_provider.clone(),
3978                            model: default_model.clone(),
3979                        },
3980                        selected_selection: routing_selection.selection.clone(),
3981                        decision,
3982                        timestamp: OffsetDateTime::now_utc(),
3983                    },
3984                ))
3985                .await;
3986            }
3987            provider = routing_selection.selection.provider.clone();
3988            model = routing_selection.selection.model.clone();
3989            let engine = self.engine_for(&provider)?;
3990            let capabilities = engine.capabilities();
3991            model_profile = model_profile_for_provider_model(&cfg, &provider, &model);
3992            let tools = if capabilities.tool_calls {
3993                if model == routing_tools_model {
3994                    routing_tools.clone()
3995                } else {
3996                    self.filtered_tool_specs(
3997                        &cfg,
3998                        &model,
3999                        model_profile.as_ref(),
4000                        &thread_overrides.tool_allowlist,
4001                        &thread_overrides.external_tools,
4002                    )
4003                }
4004            } else {
4005                Vec::new()
4006            };
4007            let parallel_tool_calls = parallel_tool_calls_for_model(&cfg, &model);
4008            let tool_choice = if tools.is_empty() {
4009                ToolChoice::None
4010            } else {
4011                ToolChoice::Auto
4012            };
4013            let summary_selection = ModelSelection {
4014                provider: provider.clone(),
4015                model: model.clone(),
4016            };
4017            if model_switch_summary_selection.as_ref() != Some(&summary_selection) {
4018                if let Some(summary) = model_switch_summary(
4019                    &transcript,
4020                    model_profile.as_ref(),
4021                    &provider,
4022                    &model,
4023                    &tools,
4024                ) {
4025                    let item = TranscriptItem::UserMessage(UserMessage::text(summary));
4026                    self.persist_turn_item(&req.thread_id, &turn_id, &item)
4027                        .await?;
4028                    transcript.push(item);
4029                }
4030                model_switch_summary_selection = Some(summary_selection);
4031            }
4032
4033            if !capabilities.image_input && transcript_has_images(&transcript) {
4034                self.fail_turn_with_error(
4035                    &req.thread_id,
4036                    &turn_id,
4037                    format!("provider {provider} does not support image input"),
4038                )
4039                .await?;
4040                return Ok(TurnRunOutcome::Stopped);
4041            }
4042            transcript = self
4043                .compact_transcript_if_needed(
4044                    &req.thread_id,
4045                    &turn_id,
4046                    &provider,
4047                    &model,
4048                    transcript,
4049                    self.compaction_options_for_turn(&req.thread_id, !compacted_this_turn),
4050                )
4051                .await?;
4052            compacted_this_turn = compacted_this_turn
4053                || transcript
4054                    .iter()
4055                    .any(crate::compaction::is_compaction_boundary);
4056
4057            let speed_policy_decision =
4058                speed_policy.decision(runtime_profile, &model, &cfg.speed_policy);
4059            let request_reasoning = reasoning_from_decision(
4060                speed_policy_decision.as_ref(),
4061                routing_selection
4062                    .reasoning
4063                    .clone()
4064                    .unwrap_or_else(|| reasoning_for_model(&cfg, &model)),
4065            );
4066            self.active_turn_selections.write().await.insert(
4067                turn_id.clone(),
4068                ModelSelectionMode::manual(
4069                    provider.clone(),
4070                    model.clone(),
4071                    request_reasoning.level.clone(),
4072                ),
4073            );
4074            if let Some(limit) = reliability.record_model_call(
4075                &cfg.reliability,
4076                runtime_profile == RuntimeProfile::Interactive,
4077            ) {
4078                self.fail_turn_due_to_reliability_limit(
4079                    &req.thread_id,
4080                    &turn_id,
4081                    &provider,
4082                    &model,
4083                    limit,
4084                    &transcript,
4085                )
4086                .await?;
4087                return Ok(TurnRunOutcome::Stopped);
4088            }
4089            self.emit(RoderEvent::InferenceStarted(InferenceStarted {
4090                thread_id: req.thread_id.clone(),
4091                turn_id: turn_id.clone(),
4092                engine_id: engine.id(),
4093                model: ModelSelection {
4094                    provider: provider.clone(),
4095                    model: model.clone(),
4096                },
4097                reasoning: request_reasoning.clone(),
4098                speed_policy: speed_policy_decision.clone(),
4099                deadline_remaining_seconds: deadline_remaining_seconds(turn_deadline),
4100                timestamp: OffsetDateTime::now_utc(),
4101            }))
4102            .await;
4103
4104            let mut instructions = req.instructions.clone();
4105            if let Some(extra) = &thread_overrides.developer_instructions {
4106                instructions = apply_thread_developer_instructions(instructions, extra);
4107            }
4108            if let Some(context) = req.developer_context.as_deref() {
4109                instructions = apply_turn_developer_context(instructions, context);
4110            }
4111            let mut instructions = apply_runtime_profile(instructions, runtime_profile);
4112            if let Some(profile) = &model_profile {
4113                instructions = apply_model_instruction_overlay(instructions, profile);
4114            }
4115            if req.task_ledger_required
4116                && runtime_profile == RuntimeProfile::Eval
4117                && !transcript_has_task_ledger(&transcript)
4118            {
4119                instructions = apply_task_ledger_required(instructions);
4120            }
4121            if effective_policy_mode == PolicyMode::Plan {
4122                instructions = apply_plan_mode(instructions);
4123            }
4124            if agent_swarm_mode_active {
4125                instructions = apply_agent_swarm_mode(instructions);
4126            }
4127            // Ultra mode (any model) enables proactive multi-agent policy.
4128            // Codex Sol/Terra Ultra effort still does too without requiring the
4129            // mode flag. Models that advertise Ultra effort keep the
4130            // explicit-request-only policy on lower efforts when ultra mode is
4131            // off; other models get multi-agent policy only when ultra mode is on.
4132            let model_has_ultra_effort =
4133                model_supports_reasoning_effort(&model, REASONING_ULTRA);
4134            let effort_is_ultra =
4135                request_reasoning.level.as_deref() == Some(REASONING_ULTRA);
4136            let proactive_multi_agent = ultra_mode_active || effort_is_ultra;
4137            if ultra_mode_active || model_has_ultra_effort {
4138                instructions =
4139                    apply_codex_multi_agent_mode(instructions, proactive_multi_agent);
4140            }
4141            instructions = self
4142                .goals
4143                .apply_goal_instructions(&req.thread_id, instructions)
4144                .await?;
4145            instructions = apply_parallel_web_tools(
4146                instructions,
4147                tools.iter().map(|spec| spec.name.as_str()),
4148            );
4149            let mut request_metadata = serde_json::json!({});
4150            if let Some(decision) = &speed_policy_decision {
4151                request_metadata["speedPolicy"] = serde_json::json!(decision);
4152            }
4153            if let Some(decision) = routing_selection.decision.as_ref() {
4154                request_metadata["inferenceRouting"] = serde_json::json!(decision);
4155            }
4156            if let Some(remaining) = deadline_remaining_seconds(turn_deadline) {
4157                request_metadata["deadlineRemainingSeconds"] = serde_json::json!(remaining);
4158            }
4159            if let Some(profile) = &model_profile {
4160                request_metadata["modelProfile"] = serde_json::json!({
4161                    "model": profile.model,
4162                    "providerFamily": profile.provider_family,
4163                    "editTool": profile.edit_tool,
4164                    "schemaPolicy": profile.schema_policy,
4165                    "instructionOverlay": profile.instruction_overlay,
4166                    "parallelToolCalls": profile.parallel_tool_calls,
4167                    "autoCompactTokenLimit": profile.auto_compact_token_limit,
4168                });
4169            }
4170            let task_ledger_required_this_round = req.task_ledger_required
4171                && runtime_profile == RuntimeProfile::Eval
4172                && !deadline_finalization_requested
4173                && !transcript_has_task_ledger(&transcript);
4174            let task_ledger_tools = (capabilities.tool_calls && task_ledger_required_this_round)
4175                .then(|| self.task_ledger_tool_specs(model_profile.as_ref()))
4176                .filter(|tools| !tools.is_empty());
4177            let request_tools = if deadline_finalization_requested {
4178                Vec::new()
4179            } else if let Some(ledger_tools) = &task_ledger_tools {
4180                ledger_tools.clone()
4181            } else {
4182                tools.clone()
4183            };
4184            let request_tool_choice = if deadline_finalization_requested {
4185                ToolChoice::None
4186            } else if task_ledger_tools.is_some() {
4187                ToolChoice::Specific(TASK_LEDGER_TOOL_NAME.to_string())
4188            } else {
4189                tool_choice.clone()
4190            };
4191            if deadline_finalization_requested {
4192                request_metadata["deadlineFinalization"] = serde_json::json!({
4193                    "reserveSeconds": deadline_finalization_reserve,
4194                    "remainingSeconds": deadline_remaining_seconds(turn_deadline),
4195                });
4196            } else if deadline_scoreable_completion_requested {
4197                request_metadata["deadlineScoreableCompletion"] = serde_json::json!({
4198                    "reserveSeconds": deadline_finalization_reserve,
4199                    "remainingSeconds": deadline_remaining_seconds(turn_deadline),
4200                });
4201            }
4202            let request = AgentInferenceRequest {
4203                model: ModelSelection {
4204                    provider: provider.clone(),
4205                    model: model.clone(),
4206                },
4207                instructions,
4208                transcript: transcript.clone(),
4209                tools: request_tools,
4210                tool_choice: request_tool_choice,
4211                reasoning: request_reasoning,
4212                output: OutputConfig::default(),
4213                runtime: RuntimeHints {
4214                    auto_compact_token_limit: server_side_compaction_threshold(&cfg, &model),
4215                    profile: runtime_profile,
4216                    parallel_tool_calls: Some(parallel_tool_calls),
4217                    hosted_web_search: cfg.hosted_web_search.clone(),
4218                    tool_search: tool_search_for_provider_model(&cfg, &provider, &model),
4219                    speed_policy: speed_policy_decision,
4220                    reliability: Some(cfg.reliability.clone().into()),
4221                    deadline_remaining_seconds: deadline_remaining_seconds(turn_deadline),
4222                    ..RuntimeHints::default()
4223                },
4224                metadata: request_metadata,
4225            };
4226
4227            let ctx = InferenceTurnContext {
4228                thread_id: &req.thread_id,
4229                turn_id: &turn_id,
4230                tool_executor: Some(std::sync::Arc::new(
4231                    crate::tool_execution::RuntimeTurnToolExecutor {
4232                        runtime: Arc::clone(self),
4233                        thread_id: req.thread_id.clone(),
4234                        turn_id: turn_id.clone(),
4235                        workspace: Some(workspace.clone()),
4236                        deadline: turn_deadline,
4237                    },
4238                )),
4239            };
4240            let stream_future = engine.stream_turn(ctx, request);
4241            let mut stream = if let Some((deadline, timeout_action)) = inference_timeout_deadline(
4242                turn_deadline,
4243                runtime_profile,
4244                req.task_ledger_required,
4245                deadline_finalization_reserve,
4246                deadline_finalization_requested || deadline_scoreable_completion_requested,
4247                task_ledger_scoreable_checkpoints,
4248                &transcript,
4249            ) {
4250                match tokio::time::timeout_at(deadline_instant(deadline), stream_future).await {
4251                    Ok(stream) => match stream {
4252                        Ok(stream) => stream,
4253                        Err(err) => {
4254                            let error = err.to_string();
4255                            if self
4256                                .try_recover_from_context_limit(
4257                                    &req.thread_id,
4258                                    &turn_id,
4259                                    &provider,
4260                                    &model,
4261                                    &error,
4262                                    &mut transcript,
4263                                    &mut compacted_this_turn,
4264                                    &mut context_limit_recovery_attempts,
4265                                )
4266                                .await?
4267                            {
4268                                continue 'tool_rounds;
4269                            }
4270                            return Err(err);
4271                        }
4272                    },
4273                    Err(_) => {
4274                        if runtime_profile == RuntimeProfile::Eval
4275                            && !deadline_finalization_requested
4276                        {
4277                            let remaining = deadline_remaining_seconds(turn_deadline).unwrap_or(0);
4278                            if timeout_action == InferenceTimeoutAction::ScoreableCheckpoint
4279                                && task_ledger_scoreable_checkpoints
4280                                    < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
4281                                && let Some(prompt) = task_ledger_completion_prompt(&transcript)
4282                            {
4283                                task_ledger_scoreable_checkpoints += 1;
4284                                let item = TranscriptItem::UserMessage(UserMessage::text(
4285                                    task_ledger_scoreable_checkpoint_prompt(remaining, &prompt),
4286                                ));
4287                                self.persist_turn_item(&req.thread_id, &turn_id, &item)
4288                                    .await?;
4289                                transcript.push(item);
4290                                continue 'tool_rounds;
4291                            }
4292                            self.start_deadline_finalization(
4293                                &req.thread_id,
4294                                &turn_id,
4295                                &mut transcript,
4296                                remaining,
4297                            )
4298                            .await?;
4299                            deadline_finalization_requested = true;
4300                            continue 'tool_rounds;
4301                        }
4302                        self.fail_turn_due_to_deadline(
4303                            &req.thread_id,
4304                            &turn_id,
4305                            deadline,
4306                            &transcript,
4307                        )
4308                        .await?;
4309                        return Ok(TurnRunOutcome::Stopped);
4310                    }
4311                }
4312            } else {
4313                match stream_future.await {
4314                    Ok(stream) => stream,
4315                    Err(err) => {
4316                        let error = err.to_string();
4317                        if self
4318                            .try_recover_from_context_limit(
4319                                &req.thread_id,
4320                                &turn_id,
4321                                &provider,
4322                                &model,
4323                                &error,
4324                                &mut transcript,
4325                                &mut compacted_this_turn,
4326                                &mut context_limit_recovery_attempts,
4327                            )
4328                            .await?
4329                        {
4330                            continue 'tool_rounds;
4331                        }
4332                        return Err(err);
4333                    }
4334                }
4335            };
4336            let mut assistant_text = String::new();
4337            let mut phase_messages = Vec::<AssistantMessage>::new();
4338            let mut reasoning_text = String::new();
4339            let mut tool_calls = Vec::new();
4340            let mut provider_metadata = None;
4341            // Captured from mid-stream Compaction(completed) events. Codex/OpenAI
4342            // sometimes abort the SSE stream after emitting the compaction item
4343            // ("error decoding response body") before response.completed, so we
4344            // must not rely solely on ProviderMetadata for the boundary.
4345            let mut stream_compaction_item: Option<serde_json::Value> = None;
4346
4347            loop {
4348                let next = if let Some((deadline, timeout_action)) = inference_timeout_deadline(
4349                    turn_deadline,
4350                    runtime_profile,
4351                    req.task_ledger_required,
4352                    deadline_finalization_reserve,
4353                    deadline_finalization_requested || deadline_scoreable_completion_requested,
4354                    task_ledger_scoreable_checkpoints,
4355                    &transcript,
4356                ) {
4357                    match tokio::time::timeout_at(deadline_instant(deadline), stream.next()).await {
4358                        Ok(next) => next,
4359                        Err(_) => {
4360                            if runtime_profile == RuntimeProfile::Eval
4361                                && !deadline_finalization_requested
4362                            {
4363                                let remaining =
4364                                    deadline_remaining_seconds(turn_deadline).unwrap_or(0);
4365                                if timeout_action == InferenceTimeoutAction::ScoreableCheckpoint
4366                                    && task_ledger_scoreable_checkpoints
4367                                        < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
4368                                    && let Some(prompt) = task_ledger_completion_prompt(&transcript)
4369                                {
4370                                    task_ledger_scoreable_checkpoints += 1;
4371                                    let item = TranscriptItem::UserMessage(UserMessage::text(
4372                                        task_ledger_scoreable_checkpoint_prompt(remaining, &prompt),
4373                                    ));
4374                                    self.persist_turn_item(&req.thread_id, &turn_id, &item)
4375                                        .await?;
4376                                    transcript.push(item);
4377                                    continue 'tool_rounds;
4378                                }
4379                                self.start_deadline_finalization(
4380                                    &req.thread_id,
4381                                    &turn_id,
4382                                    &mut transcript,
4383                                    remaining,
4384                                )
4385                                .await?;
4386                                deadline_finalization_requested = true;
4387                                continue 'tool_rounds;
4388                            }
4389                            self.fail_turn_due_to_deadline(
4390                                &req.thread_id,
4391                                &turn_id,
4392                                deadline,
4393                                &transcript,
4394                            )
4395                            .await?;
4396                            return Ok(TurnRunOutcome::Stopped);
4397                        }
4398                    }
4399                } else {
4400                    stream.next().await
4401                };
4402                let Some(res) = next else {
4403                    break;
4404                };
4405                let event = match res {
4406                    Ok(event) => event,
4407                    Err(err) => {
4408                        let error = err.to_string();
4409                        if runtime_profile == RuntimeProfile::Eval
4410                            && !deadline_finalization_requested
4411                            && let Some(cause) = provider_stream_retry_cause(&error)
4412                        {
4413                            let retry_attempt = provider_stream_retry_attempts.saturating_add(1);
4414                            let policy: ReliabilityRequestPolicy = cfg.reliability.clone().into();
4415                            if retry_attempt < policy.provider_retry_max_attempts {
4416                                provider_stream_retry_attempts = retry_attempt;
4417                                let delay_ms = provider_retry_delay_ms(&policy, retry_attempt);
4418                                self.emit(RoderEvent::ReliabilityRetryRecorded(
4419                                    ReliabilityRetryRecorded {
4420                                        context: ReliabilityContext {
4421                                            thread_id: req.thread_id.clone(),
4422                                            turn_id: turn_id.clone(),
4423                                            provider: Some(provider.clone()),
4424                                            model: Some(model.clone()),
4425                                            ..ReliabilityContext::default()
4426                                        },
4427                                        error_class: ReliabilityErrorClass::ProviderError,
4428                                        decision: ReliabilityRetryDecision::Retry,
4429                                        attempt: retry_attempt,
4430                                        max_attempts: policy.provider_retry_max_attempts,
4431                                        delay_ms: Some(delay_ms),
4432                                        details: ReliabilityDetails::redacted(format!(
4433                                            "{cause}: {error}"
4434                                        )),
4435                                        timestamp: OffsetDateTime::now_utc(),
4436                                    },
4437                                ))
4438                                .await;
4439                                if delay_ms > 0 {
4440                                    tokio::time::sleep(std::time::Duration::from_millis(delay_ms))
4441                                        .await;
4442                                }
4443                                continue 'tool_rounds;
4444                            }
4445                        }
4446                        if self
4447                            .try_recover_from_context_limit(
4448                                &req.thread_id,
4449                                &turn_id,
4450                                &provider,
4451                                &model,
4452                                &error,
4453                                &mut transcript,
4454                                &mut compacted_this_turn,
4455                                &mut context_limit_recovery_attempts,
4456                            )
4457                            .await?
4458                        {
4459                            continue 'tool_rounds;
4460                        }
4461                        self.emit(RoderEvent::TurnFailed(TurnFailed {
4462                            thread_id: req.thread_id.clone(),
4463                            turn_id: turn_id.clone(),
4464                            error: error.clone(),
4465                            error_kind: None,
4466                            usage: None,
4467                            timestamp: OffsetDateTime::now_utc(),
4468                        }))
4469                        .await;
4470                        self.complete_team_member_turn_with_result(
4471                            &req.thread_id,
4472                            &turn_id,
4473                            TeamMemberStatus::Failed,
4474                            (!assistant_text.trim().is_empty()).then(|| assistant_text.clone()),
4475                            Some(error),
4476                        )
4477                        .await?;
4478                        return Err(err);
4479                    }
4480                };
4481
4482                let inference_timestamp = OffsetDateTime::now_utc();
4483                self.emit(RoderEvent::InferenceEventReceived(InferenceEventReceived {
4484                    thread_id: req.thread_id.clone(),
4485                    turn_id: turn_id.clone(),
4486                    event: event.clone(),
4487                    timestamp: inference_timestamp,
4488                }))
4489                .await;
4490
4491                match event {
4492                    InferenceEvent::MessageDelta(delta) => {
4493                        if let Some((team_id, member)) =
4494                            self.teams.member_for_thread(&req.thread_id).await
4495                        {
4496                            self.emit(RoderEvent::TeamMemberMessageDelta(TeamMemberMessageDelta {
4497                                team_id,
4498                                member_id: member.id,
4499                                member_thread_id: req.thread_id.clone(),
4500                                turn_id: turn_id.clone(),
4501                                delta: delta.text.clone(),
4502                                timestamp: OffsetDateTime::now_utc(),
4503                            }))
4504                            .await;
4505                        }
4506                        if is_final_answer_phase(delta.phase.as_deref()) {
4507                            assistant_text.push_str(&delta.text);
4508                        } else if let Some(last) = phase_messages.last_mut()
4509                            && last.phase == delta.phase
4510                        {
4511                            last.text.push_str(&delta.text);
4512                        } else {
4513                            phase_messages.push(AssistantMessage {
4514                                text: delta.text,
4515                                phase: delta.phase,
4516                            });
4517                        }
4518                    }
4519                    InferenceEvent::ReasoningDelta(delta) => reasoning_text.push_str(&delta.text),
4520                    InferenceEvent::ToolCallCompleted(call) => tool_calls.push(call),
4521                    InferenceEvent::Failed(failure) => {
4522                        speed_policy.record_failure();
4523                        let error = failure.message;
4524                        if self
4525                            .try_recover_from_context_limit(
4526                                &req.thread_id,
4527                                &turn_id,
4528                                &provider,
4529                                &model,
4530                                &error,
4531                                &mut transcript,
4532                                &mut compacted_this_turn,
4533                                &mut context_limit_recovery_attempts,
4534                            )
4535                            .await?
4536                        {
4537                            continue 'tool_rounds;
4538                        }
4539                        self.persist_turn_item(
4540                            &req.thread_id,
4541                            &turn_id,
4542                            &TranscriptItem::Error(ErrorRecord {
4543                                message: error.clone(),
4544                            }),
4545                        )
4546                        .await?;
4547                        self.emit(RoderEvent::TurnFailed(TurnFailed {
4548                            thread_id: req.thread_id.clone(),
4549                            turn_id: turn_id.clone(),
4550                            error: error.clone(),
4551                            error_kind: None,
4552                            usage: None,
4553                            timestamp: OffsetDateTime::now_utc(),
4554                        }))
4555                        .await;
4556                        self.complete_team_member_turn_with_result(
4557                            &req.thread_id,
4558                            &turn_id,
4559                            TeamMemberStatus::Failed,
4560                            (!assistant_text.trim().is_empty()).then(|| assistant_text.clone()),
4561                            Some(error),
4562                        )
4563                        .await?;
4564                        return Ok(TurnRunOutcome::Stopped);
4565                    }
4566                    InferenceEvent::Usage(usage) => {
4567                        turn_usage.add_assign(&usage);
4568                    }
4569                    InferenceEvent::Completed(metadata) => {
4570                        turn_finish_reason = metadata
4571                            .stop_reason
4572                            .as_deref()
4573                            .map(finish_reason_from_stop_reason);
4574                    }
4575                    InferenceEvent::Compaction(progress) => {
4576                        // Persist the boundary as soon as the provider emits a
4577                        // completed compaction item. ChatGPT Codex often aborts
4578                        // the SSE stream right after ("error decoding response
4579                        // body"), so waiting for ProviderMetadata loses the
4580                        // boundary and every subsequent request re-compacts.
4581                        if progress.status == "completed"
4582                            && let Some(item) = progress.item
4583                        {
4584                            let already = stream_compaction_item
4585                                .as_ref()
4586                                .and_then(|existing| {
4587                                    existing.get("id").and_then(serde_json::Value::as_str)
4588                                })
4589                                .zip(item.get("id").and_then(serde_json::Value::as_str))
4590                                .is_some_and(|(a, b)| a == b);
4591                            if !already {
4592                                stream_compaction_item = Some(item.clone());
4593                                let boundary = TranscriptItem::ProviderMetadata(
4594                                    crate::compaction::provider_metadata_from_compaction_item(item),
4595                                );
4596                                self.persist_turn_item(&req.thread_id, &turn_id, &boundary)
4597                                    .await?;
4598                                transcript.push(boundary);
4599                                transcript =
4600                                    crate::compaction::trim_to_last_compaction_boundary(transcript);
4601                                compacted_this_turn = true;
4602                            }
4603                        }
4604                    }
4605                    InferenceEvent::HostedToolCallStarted(_)
4606                    | InferenceEvent::HostedToolCallCompleted(_)
4607                    | InferenceEvent::ToolCallStarted(_)
4608                    | InferenceEvent::ToolCallDelta(_) => {}
4609                    InferenceEvent::ProviderMetadata(mut metadata) => {
4610                        if let Some(compaction_item) = stream_compaction_item.as_ref() {
4611                            let _ = crate::compaction::ensure_provider_metadata_compaction(
4612                                &mut metadata,
4613                                compaction_item,
4614                            );
4615                        }
4616                        provider_metadata = Some(metadata);
4617                    }
4618                }
4619            }
4620
4621            speed_policy.record_model_output(
4622                !assistant_text.is_empty() || !phase_messages.is_empty(),
4623                tool_calls.len(),
4624            );
4625            if tool_calls.is_empty() {
4626                let steers = self.drain_turn_steers(&turn_id).await;
4627                if !steers.is_empty() {
4628                    for message in phase_messages {
4629                        let item = TranscriptItem::AssistantMessage(message);
4630                        self.persist_turn_item(&req.thread_id, &turn_id, &item)
4631                            .await?;
4632                        transcript.push(item);
4633                        self.persist_model_profile_segment(
4634                            &req.thread_id,
4635                            &turn_id,
4636                            model_profile.as_ref(),
4637                            &provider,
4638                            &model,
4639                            "assistant",
4640                        )
4641                        .await?;
4642                    }
4643                    if !assistant_text.is_empty() {
4644                        let assistant = TranscriptItem::AssistantMessage(AssistantMessage {
4645                            text: assistant_text,
4646                            phase: Some(FINAL_ANSWER_PHASE.to_string()),
4647                        });
4648                        self.persist_turn_item(&req.thread_id, &turn_id, &assistant)
4649                            .await?;
4650                        transcript.push(assistant);
4651                        self.persist_model_profile_segment(
4652                            &req.thread_id,
4653                            &turn_id,
4654                            model_profile.as_ref(),
4655                            &provider,
4656                            &model,
4657                            "assistant",
4658                        )
4659                        .await?;
4660                    }
4661                    if let Some(metadata) = provider_metadata {
4662                        let had_provider_compaction =
4663                            crate::compaction::provider_metadata_has_compaction(&metadata);
4664                        let item = TranscriptItem::ProviderMetadata(metadata);
4665                        self.persist_turn_item(&req.thread_id, &turn_id, &item)
4666                            .await?;
4667                        transcript.push(item);
4668                        if had_provider_compaction {
4669                            // Server-side compaction replaced the prior window;
4670                            // drop pre-boundary items so the next request does
4671                            // not re-send (and re-compact) the full history.
4672                            transcript =
4673                                crate::compaction::trim_to_last_compaction_boundary(transcript);
4674                            compacted_this_turn = true;
4675                        }
4676                    }
4677                    self.append_steers(&req, &turn_id, &mut transcript, steers)
4678                        .await?;
4679                    continue;
4680                }
4681                if !deadline_finalization_requested
4682                    && req.task_ledger_required
4683                    && runtime_profile == RuntimeProfile::Eval
4684                    && task_ledger_completion_reminders < TASK_LEDGER_COMPLETION_REMINDER_LIMIT
4685                    && (!assistant_text.trim().is_empty() || !phase_messages.is_empty())
4686                    && let Some(prompt) = task_ledger_completion_prompt(&transcript)
4687                {
4688                    task_ledger_completion_reminders += 1;
4689                    let item = TranscriptItem::UserMessage(UserMessage::text(prompt));
4690                    self.persist_turn_item(&req.thread_id, &turn_id, &item)
4691                        .await?;
4692                    transcript.push(item);
4693                    continue;
4694                }
4695                if !deadline_finalization_requested
4696                    && let Some(prompt) = verification_gate.blocking_prompt()
4697                {
4698                    speed_policy.record_verification_required();
4699                    self.emit(RoderEvent::VerificationRequired(VerificationRequired {
4700                        thread_id: req.thread_id.clone(),
4701                        turn_id: turn_id.clone(),
4702                        reason: verification_gate.reason(),
4703                        changed_files: verification_gate.changed_files(),
4704                        tool_evidence: verification_gate.tool_evidence.clone(),
4705                        tests_run: verification_gate.tests_run.clone(),
4706                        open_gaps: verification_gate.open_gaps.clone(),
4707                        timestamp: OffsetDateTime::now_utc(),
4708                    }))
4709                    .await;
4710                    let item = TranscriptItem::UserMessage(UserMessage::text(prompt));
4711                    self.persist_turn_item(&req.thread_id, &turn_id, &item)
4712                        .await?;
4713                    transcript.push(item);
4714                    continue;
4715                }
4716                // Loop persistence nudge: in non-interactive/eval runs, when the
4717                // model returns a final message with no tool calls, gently prompt
4718                // it to keep going (bounded by `empty_tool_call_nudges`) before
4719                // ending the turn. Gated to non-interactive/eval so interactive
4720                // chat completions are never delayed, and only fires when the
4721                // model actually produced a final answer to re-examine.
4722                if !deadline_finalization_requested
4723                    && runtime_profile != RuntimeProfile::Interactive
4724                    && cfg.reliability.empty_tool_call_nudges > 0
4725                    && empty_tool_call_nudges_used < cfg.reliability.empty_tool_call_nudges
4726                    && (!assistant_text.trim().is_empty() || !phase_messages.is_empty())
4727                {
4728                    empty_tool_call_nudges_used += 1;
4729                    let item = TranscriptItem::UserMessage(UserMessage::text(
4730                        RELIABILITY_CONTINUATION_PROMPT.to_string(),
4731                    ));
4732                    self.persist_turn_item(&req.thread_id, &turn_id, &item)
4733                        .await?;
4734                    transcript.push(item);
4735                    continue;
4736                }
4737                if deadline_finalization_requested
4738                    && assistant_text.trim().is_empty()
4739                    && phase_messages.is_empty()
4740                {
4741                    assistant_text = format!(
4742                        "Deadline finalization completed without model text. {}",
4743                        turn_partial_result(&transcript)
4744                    );
4745                }
4746                final_phase_messages = phase_messages;
4747                final_assistant_text = assistant_text;
4748                final_reasoning_text = reasoning_text;
4749                final_provider_metadata = provider_metadata;
4750                exhausted_tool_rounds = false;
4751                break;
4752            }
4753
4754            for message in phase_messages {
4755                let item = TranscriptItem::AssistantMessage(message);
4756                self.persist_turn_item(&req.thread_id, &turn_id, &item)
4757                    .await?;
4758                transcript.push(item);
4759                self.persist_model_profile_segment(
4760                    &req.thread_id,
4761                    &turn_id,
4762                    model_profile.as_ref(),
4763                    &provider,
4764                    &model,
4765                    "assistant",
4766                )
4767                .await?;
4768            }
4769            if !assistant_text.is_empty() {
4770                transcript.push(TranscriptItem::AssistantMessage(AssistantMessage {
4771                    text: assistant_text,
4772                    phase: Some(FINAL_ANSWER_PHASE.to_string()),
4773                }));
4774                self.persist_model_profile_segment(
4775                    &req.thread_id,
4776                    &turn_id,
4777                    model_profile.as_ref(),
4778                    &provider,
4779                    &model,
4780                    "assistant",
4781                )
4782                .await?;
4783            }
4784            if let Some(metadata) = provider_metadata {
4785                let had_provider_compaction =
4786                    crate::compaction::provider_metadata_has_compaction(&metadata);
4787                let item = TranscriptItem::ProviderMetadata(metadata);
4788                self.persist_turn_item(&req.thread_id, &turn_id, &item)
4789                    .await?;
4790                transcript.push(item);
4791                if had_provider_compaction {
4792                    // Server-side compaction replaced the prior window; drop
4793                    // pre-boundary items so subsequent tool rounds use the
4794                    // compact window as the next input.
4795                    transcript = crate::compaction::trim_to_last_compaction_boundary(transcript);
4796                    compacted_this_turn = true;
4797                }
4798            }
4799            // Persist CoT before tool calls so providers that require
4800            // `reasoning_content` on tool-call assistant turns (DeepSeek) can
4801            // replay it on the next request. Without this, multi-step tool
4802            // rollouts drop the thinking block and the API returns 400.
4803            if !reasoning_text.is_empty() {
4804                let item = TranscriptItem::ReasoningSummary(ReasoningSummary {
4805                    text: std::mem::take(&mut reasoning_text),
4806                });
4807                self.persist_turn_item(&req.thread_id, &turn_id, &item)
4808                    .await?;
4809                transcript.push(item);
4810            }
4811            for call in &tool_calls {
4812                let tool_item = TranscriptItem::ToolCall(ToolCallRecord {
4813                    id: call.id.clone(),
4814                    name: call.name.clone(),
4815                    arguments: call.arguments.clone(),
4816                });
4817                self.persist_turn_item(&req.thread_id, &turn_id, &tool_item)
4818                    .await?;
4819                transcript.push(tool_item);
4820                self.persist_model_profile_segment(
4821                    &req.thread_id,
4822                    &turn_id,
4823                    model_profile.as_ref(),
4824                    &provider,
4825                    &model,
4826                    "tool_call",
4827                )
4828                .await?;
4829            }
4830            if let Some(deadline) = turn_deadline
4831                && deadline_expired(deadline)
4832            {
4833                self.fail_turn_due_to_deadline(&req.thread_id, &turn_id, deadline, &transcript)
4834                    .await?;
4835                return Ok(TurnRunOutcome::Stopped);
4836            }
4837            let results = self
4838                .route_tool_calls(
4839                    &req.thread_id,
4840                    &turn_id,
4841                    tool_calls,
4842                    parallel_tool_calls,
4843                    Some(workspace.as_str()),
4844                    turn_deadline,
4845                )
4846                .await?;
4847            let reliability_limit = reliability.record_tool_results(
4848                &cfg.reliability,
4849                &results,
4850                runtime_profile == RuntimeProfile::Interactive,
4851            );
4852            for result in results {
4853                verification_gate.record_tool_result(&result);
4854                transcript.push(TranscriptItem::ToolResult(result));
4855                self.persist_model_profile_segment(
4856                    &req.thread_id,
4857                    &turn_id,
4858                    model_profile.as_ref(),
4859                    &provider,
4860                    &model,
4861                    "tool_result",
4862                )
4863                .await?;
4864            }
4865            if let Some(limit) = reliability_limit {
4866                if limit.decision == ReliabilityLimitDecision::RequestContinuation {
4867                    // Loop persistence: rather than ending the turn, reset the
4868                    // consecutive-failure counter, nudge the model to keep going,
4869                    // and continue the round loop. Bounded by the per-turn tool
4870                    // failure ceiling, `max_model_calls_per_turn`, and
4871                    // `MAX_TOOL_ROUNDS_PER_TURN`.
4872                    self.record_reliability_limit_continuation(
4873                        &req.thread_id,
4874                        &turn_id,
4875                        &provider,
4876                        &model,
4877                        limit,
4878                    )
4879                    .await;
4880                    reliability.reset_consecutive_failures();
4881                    let nudge = TranscriptItem::UserMessage(UserMessage::text(
4882                        RELIABILITY_CONTINUATION_PROMPT.to_string(),
4883                    ));
4884                    self.persist_turn_item(&req.thread_id, &turn_id, &nudge)
4885                        .await?;
4886                    transcript.push(nudge);
4887                } else {
4888                    self.fail_turn_due_to_reliability_limit(
4889                        &req.thread_id,
4890                        &turn_id,
4891                        &provider,
4892                        &model,
4893                        limit,
4894                        &transcript,
4895                    )
4896                    .await?;
4897                    return Ok(TurnRunOutcome::Stopped);
4898                }
4899            }
4900            transcript = self
4901                .compact_transcript_if_needed(
4902                    &req.thread_id,
4903                    &turn_id,
4904                    &provider,
4905                    &model,
4906                    transcript,
4907                    self.compaction_options_for_turn(&req.thread_id, !compacted_this_turn),
4908                )
4909                .await?;
4910            compacted_this_turn = compacted_this_turn
4911                || transcript
4912                    .iter()
4913                    .any(crate::compaction::is_compaction_boundary);
4914        }
4915
4916        if exhausted_tool_rounds {
4917            let message =
4918                format!("tool call limit reached after {MAX_TOOL_ROUNDS_PER_TURN} rounds");
4919            self.persist_turn_item(
4920                &req.thread_id,
4921                &turn_id,
4922                &TranscriptItem::Error(ErrorRecord {
4923                    message: message.clone(),
4924                }),
4925            )
4926            .await?;
4927            self.emit(RoderEvent::TurnFailed(TurnFailed {
4928                thread_id: req.thread_id.clone(),
4929                turn_id: turn_id.clone(),
4930                error: message.clone(),
4931                error_kind: None,
4932                usage: None,
4933                timestamp: OffsetDateTime::now_utc(),
4934            }))
4935            .await;
4936            self.complete_team_member_turn_with_result(
4937                &req.thread_id,
4938                &turn_id,
4939                TeamMemberStatus::Failed,
4940                None,
4941                Some(message),
4942            )
4943            .await?;
4944            return Ok(TurnRunOutcome::Stopped);
4945        }
4946
4947        if !final_reasoning_text.is_empty() {
4948            self.persist_turn_item(
4949                &req.thread_id,
4950                &turn_id,
4951                &TranscriptItem::ReasoningSummary(ReasoningSummary {
4952                    text: final_reasoning_text,
4953                }),
4954            )
4955            .await?;
4956        }
4957        for message in final_phase_messages {
4958            self.persist_turn_item(
4959                &req.thread_id,
4960                &turn_id,
4961                &TranscriptItem::AssistantMessage(message),
4962            )
4963            .await?;
4964            self.persist_model_profile_segment(
4965                &req.thread_id,
4966                &turn_id,
4967                model_profile.as_ref(),
4968                &provider,
4969                &model,
4970                "assistant",
4971            )
4972            .await?;
4973        }
4974        if !final_assistant_text.is_empty() {
4975            self.persist_turn_item(
4976                &req.thread_id,
4977                &turn_id,
4978                &TranscriptItem::AssistantMessage(AssistantMessage {
4979                    text: final_assistant_text.clone(),
4980                    phase: Some(FINAL_ANSWER_PHASE.to_string()),
4981                }),
4982            )
4983            .await?;
4984            self.persist_model_profile_segment(
4985                &req.thread_id,
4986                &turn_id,
4987                model_profile.as_ref(),
4988                &provider,
4989                &model,
4990                "assistant",
4991            )
4992            .await?;
4993        }
4994        if let Some(metadata) = final_provider_metadata {
4995            self.persist_turn_item(
4996                &req.thread_id,
4997                &turn_id,
4998                &TranscriptItem::ProviderMetadata(metadata),
4999            )
5000            .await?;
5001        }
5002
5003        let turn_usage_tokens = turn_usage.total_tokens as i64;
5004        let completed_usage = (!turn_usage.is_empty()).then_some(turn_usage.clone());
5005        self.record_thread_usage_metadata(&req.thread_id, &turn_usage)
5006            .await?;
5007        self.goals
5008            .account_turn_usage(
5009                &req.thread_id,
5010                turn_usage_tokens,
5011                OffsetDateTime::now_utc() - turn_started_at,
5012            )
5013            .await?;
5014        let (cleanup, ownership) = self.await_provider_turn_cleanup(&turn_id).await;
5015        self.record_turn_lifecycle_with_ownership(
5016            req.thread_id.clone(),
5017            turn_id.clone(),
5018            TurnLifecycleState::Completed,
5019            cleanup,
5020            None,
5021            ownership,
5022        )
5023        .await;
5024        self.emit(RoderEvent::TurnCompleted(TurnCompleted {
5025            thread_id: req.thread_id.clone(),
5026            turn_id: turn_id.clone(),
5027            usage: completed_usage,
5028            finish_reason: turn_finish_reason,
5029            timestamp: OffsetDateTime::now_utc(),
5030        }))
5031        .await;
5032        self.complete_team_member_turn_with_result(
5033            &req.thread_id,
5034            &turn_id,
5035            TeamMemberStatus::Completed,
5036            (!final_assistant_text.is_empty()).then_some(final_assistant_text),
5037            None,
5038        )
5039        .await?;
5040        Ok(TurnRunOutcome::Completed)
5041    }
5042
5043    async fn drain_turn_steers(&self, turn_id: &TurnId) -> Vec<QueuedTurnSteer> {
5044        let Some(active) = self.active_turns.read().await.get(turn_id).cloned() else {
5045            return Vec::new();
5046        };
5047        let mut steers = active.steers.lock().await;
5048        std::mem::take(&mut *steers)
5049    }
5050
5051    async fn route_tool_calls(
5052        self: &Arc<Self>,
5053        thread_id: &ThreadId,
5054        turn_id: &TurnId,
5055        calls: Vec<ToolCallCompleted>,
5056        parallel: bool,
5057        workspace: Option<&str>,
5058        deadline: Option<OffsetDateTime>,
5059    ) -> anyhow::Result<Vec<ToolResultRecord>> {
5060        // Enforce the agent_swarm exclusivity rule (roadmap 104, Task 2):
5061        // `agent_swarm` must be the only tool call in a model response. A mixed
5062        // or multi-swarm batch is denied wholesale with actionable retry text so
5063        // the model re-issues `agent_swarm` by itself, and every tool_call_id
5064        // still gets a response (keeping the chat-completions transcript valid).
5065        if let Some(violation) = roder_api::subagents::agent_swarm_batch_violation(
5066            calls.iter().map(|call| call.name.as_str()),
5067        ) {
5068            let message = violation.deny_message();
5069            return Ok(calls
5070                .into_iter()
5071                .map(|call| ToolResultRecord {
5072                    id: call.id,
5073                    name: Some(call.name),
5074                    result: message.clone(),
5075                    display_payload: None,
5076                    is_error: true,
5077                })
5078                .collect());
5079        }
5080        // Emit swarm lifecycle events on the bus (roadmap 104, Task 1) so any
5081        // app-server/SDK/TUI client can observe a swarm as a whole; per-child
5082        // progress flows through the existing Subagent* trace events.
5083        let swarm_call = (calls.len() == 1
5084            && calls[0].name == roder_api::subagents::AGENT_SWARM_TOOL_NAME)
5085            .then(|| (calls[0].id.clone(), calls[0].arguments.clone()));
5086        if let Some((tool_id, args)) = &swarm_call {
5087            self.emit(RoderEvent::AgentSwarmStarted(
5088                roder_api::subagents::AgentSwarmStarted {
5089                    thread_id: thread_id.clone(),
5090                    turn_id: turn_id.clone(),
5091                    tool_id: tool_id.clone(),
5092                    child_count: agent_swarm_child_count(args),
5093                    timestamp: OffsetDateTime::now_utc(),
5094                },
5095            ))
5096            .await;
5097        }
5098
5099        let force_sequential = calls
5100            .iter()
5101            .any(|call| crate::agent_control_tools::is_agent_control_tool(&call.name));
5102        let results =
5103            if parallel && !force_sequential {
5104                try_join_all(calls.into_iter().map(|call| {
5105                    self.route_tool_call(thread_id, turn_id, call, workspace, deadline)
5106                }))
5107                .await
5108            } else {
5109                let mut results = Vec::with_capacity(calls.len());
5110                for call in calls {
5111                    results.push(
5112                        self.route_tool_call(thread_id, turn_id, call, workspace, deadline)
5113                            .await?,
5114                    );
5115                }
5116                Ok(results)
5117            }?;
5118
5119        if let Some((tool_id, _)) = &swarm_call
5120            && let Some(result) = results.iter().find(|result| &result.id == tool_id)
5121            && let Some((completed, failed, aborted)) = parse_swarm_counts(&result.result)
5122        {
5123            self.emit(RoderEvent::AgentSwarmCompleted(
5124                roder_api::subagents::AgentSwarmCompleted {
5125                    thread_id: thread_id.clone(),
5126                    turn_id: turn_id.clone(),
5127                    tool_id: tool_id.clone(),
5128                    completed,
5129                    failed,
5130                    aborted,
5131                    timestamp: OffsetDateTime::now_utc(),
5132                },
5133            ))
5134            .await;
5135        }
5136
5137        Ok(results)
5138    }
5139
5140
5141    /// On provider context/prompt-length failures, force-compact (and if needed
5142    /// strip the last bulky item) then retry the current tool round instead of
5143    /// failing the turn. Bounded so a permanently oversized suffix still stops.
5144    async fn try_recover_from_context_limit(
5145        &self,
5146        thread_id: &ThreadId,
5147        turn_id: &TurnId,
5148        provider: &str,
5149        model: &str,
5150        error: &str,
5151        transcript: &mut Vec<TranscriptItem>,
5152        compacted_this_turn: &mut bool,
5153        recovery_attempts: &mut u32,
5154    ) -> anyhow::Result<bool> {
5155        const MAX_CONTEXT_LIMIT_RECOVERY_ATTEMPTS: u32 = 2;
5156        if !crate::compaction::is_context_limit_failure_message(error) {
5157            return Ok(false);
5158        }
5159        if *recovery_attempts >= MAX_CONTEXT_LIMIT_RECOVERY_ATTEMPTS {
5160            return Ok(false);
5161        }
5162        *recovery_attempts = recovery_attempts.saturating_add(1);
5163
5164        let error_item = TranscriptItem::Error(ErrorRecord {
5165            message: error.to_string(),
5166        });
5167        self.persist_turn_item(thread_id, turn_id, &error_item)
5168            .await?;
5169        transcript.push(error_item);
5170
5171        // After the first failed recovery, drop the item ahead of the latest
5172        // user/error so a second compact can succeed when the suffix itself is
5173        // still oversized (e.g. a huge tool result right before the prompt).
5174        if *recovery_attempts > 1 {
5175            *transcript = crate::compaction::strip_last_message_before_error(std::mem::take(transcript));
5176        }
5177
5178        let force_options = crate::compaction::CompactionOptions {
5179            allow_repeat: true,
5180            force: true,
5181            hysteresis_baseline: None,
5182            preserve_hint: Some(
5183                "Recover from a provider context/prompt-length limit; preserve the active user goal and recent tool outcomes."
5184                    .to_string(),
5185            ),
5186        };
5187        let before_len = transcript.len();
5188        let before_tokens = crate::compaction::estimate_prompt_tokens(transcript);
5189        *transcript = self
5190            .compact_transcript_if_needed(
5191                thread_id,
5192                turn_id,
5193                provider,
5194                model,
5195                std::mem::take(transcript),
5196                force_options,
5197            )
5198            .await?;
5199        let after_tokens = crate::compaction::estimate_prompt_tokens(transcript);
5200        *compacted_this_turn = *compacted_this_turn
5201            || transcript
5202                .iter()
5203                .any(crate::compaction::is_compaction_boundary);
5204
5205        self.emit(RoderEvent::ReliabilityRetryRecorded(
5206            ReliabilityRetryRecorded {
5207                context: ReliabilityContext {
5208                    thread_id: thread_id.clone(),
5209                    turn_id: turn_id.clone(),
5210                    provider: Some(provider.to_string()),
5211                    model: Some(model.to_string()),
5212                    ..ReliabilityContext::default()
5213                },
5214                error_class: ReliabilityErrorClass::ProviderError,
5215                decision: ReliabilityRetryDecision::Retry,
5216                attempt: *recovery_attempts,
5217                max_attempts: MAX_CONTEXT_LIMIT_RECOVERY_ATTEMPTS,
5218                delay_ms: Some(0),
5219                details: ReliabilityDetails::redacted(format!(
5220                    "context_limit_recovery: compact {before_tokens}->{after_tokens} tokens (items {before_len}->{}); {error}",
5221                    transcript.len()
5222                )),
5223                timestamp: OffsetDateTime::now_utc(),
5224            },
5225        ))
5226        .await;
5227
5228        // If force-compact did not shrink anything and we still have budget,
5229        // strip once more so the next round is not identical.
5230        if after_tokens >= before_tokens && *recovery_attempts < MAX_CONTEXT_LIMIT_RECOVERY_ATTEMPTS {
5231            *transcript = crate::compaction::strip_last_message_before_error(std::mem::take(transcript));
5232        }
5233        Ok(true)
5234    }
5235
5236    async fn fail_turn_with_error(
5237        &self,
5238        thread_id: &ThreadId,
5239        turn_id: &TurnId,
5240        message: String,
5241    ) -> anyhow::Result<()> {
5242        self.persist_turn_item(
5243            thread_id,
5244            turn_id,
5245            &TranscriptItem::Error(ErrorRecord {
5246                message: message.clone(),
5247            }),
5248        )
5249        .await?;
5250        self.emit(RoderEvent::TurnFailed(TurnFailed {
5251            thread_id: thread_id.clone(),
5252            turn_id: turn_id.clone(),
5253            error: message.clone(),
5254            error_kind: None,
5255            usage: None,
5256            timestamp: OffsetDateTime::now_utc(),
5257        }))
5258        .await;
5259        self.complete_team_member_turn_with_result(
5260            thread_id,
5261            turn_id,
5262            TeamMemberStatus::Failed,
5263            None,
5264            Some(message),
5265        )
5266        .await?;
5267        Ok(())
5268    }
5269
5270    async fn fail_turn_due_to_deadline(
5271        &self,
5272        thread_id: &ThreadId,
5273        turn_id: &TurnId,
5274        deadline: OffsetDateTime,
5275        transcript: &[TranscriptItem],
5276    ) -> anyhow::Result<()> {
5277        let partial_result = turn_partial_result(transcript);
5278        self.emit(RoderEvent::TurnPartialResult(TurnPartialResult {
5279            thread_id: thread_id.clone(),
5280            turn_id: turn_id.clone(),
5281            summary: partial_result.clone(),
5282            timestamp: OffsetDateTime::now_utc(),
5283        }))
5284        .await;
5285        self.emit(RoderEvent::TurnDeadlineExceeded(TurnDeadlineExceeded {
5286            thread_id: thread_id.clone(),
5287            turn_id: turn_id.clone(),
5288            deadline,
5289            partial_result: partial_result.clone(),
5290            timestamp: OffsetDateTime::now_utc(),
5291        }))
5292        .await;
5293        let message = "turn deadline expired".to_string();
5294        self.persist_turn_item(
5295            thread_id,
5296            turn_id,
5297            &TranscriptItem::Error(ErrorRecord {
5298                message: format!("{message}: {partial_result}"),
5299            }),
5300        )
5301        .await?;
5302        self.emit(RoderEvent::TurnFailed(TurnFailed {
5303            thread_id: thread_id.clone(),
5304            turn_id: turn_id.clone(),
5305            error: message.clone(),
5306            error_kind: Some("deadline_timeout".to_string()),
5307            usage: None,
5308            timestamp: OffsetDateTime::now_utc(),
5309        }))
5310        .await;
5311        self.complete_team_member_turn_with_result(
5312            thread_id,
5313            turn_id,
5314            TeamMemberStatus::Failed,
5315            None,
5316            Some(format!("{message}: {partial_result}")),
5317        )
5318        .await?;
5319        Ok(())
5320    }
5321
5322    async fn start_deadline_finalization(
5323        &self,
5324        thread_id: &ThreadId,
5325        turn_id: &TurnId,
5326        transcript: &mut Vec<TranscriptItem>,
5327        remaining_seconds: u64,
5328    ) -> anyhow::Result<()> {
5329        let item = TranscriptItem::UserMessage(crate::deadline_policy::finalization_message(
5330            remaining_seconds,
5331        ));
5332        self.persist_turn_item(thread_id, turn_id, &item).await?;
5333        transcript.push(item);
5334        self.emit(RoderEvent::TurnPartialResult(TurnPartialResult {
5335            thread_id: thread_id.clone(),
5336            turn_id: turn_id.clone(),
5337            summary: turn_partial_result(transcript),
5338            timestamp: OffsetDateTime::now_utc(),
5339        }))
5340        .await;
5341        Ok(())
5342    }
5343
5344    /// Emits the reliability-limit event for a continuation (loop persistence)
5345    /// without failing the turn. The caller resets the failure counter and
5346    /// injects a nudge so the round loop keeps going.
5347    async fn record_reliability_limit_continuation(
5348        &self,
5349        thread_id: &ThreadId,
5350        turn_id: &TurnId,
5351        provider: &str,
5352        model: &str,
5353        limit: ReliabilityLimitHit,
5354    ) {
5355        self.emit(RoderEvent::ReliabilityLimitRecorded(
5356            ReliabilityLimitRecorded {
5357                context: ReliabilityContext {
5358                    thread_id: thread_id.clone(),
5359                    turn_id: turn_id.clone(),
5360                    tool_id: None,
5361                    tool_name: None,
5362                    provider: Some(provider.to_string()),
5363                    model: Some(model.to_string()),
5364                },
5365                error_class: limit.error_class,
5366                limit_kind: limit.limit_kind,
5367                decision: limit.decision,
5368                current: limit.current,
5369                limit: limit.limit,
5370                details: ReliabilityDetails::redacted(&limit.message),
5371                timestamp: OffsetDateTime::now_utc(),
5372            },
5373        ))
5374        .await;
5375    }
5376
5377    async fn fail_turn_due_to_reliability_limit(
5378        &self,
5379        thread_id: &ThreadId,
5380        turn_id: &TurnId,
5381        provider: &str,
5382        model: &str,
5383        limit: ReliabilityLimitHit,
5384        transcript: &[TranscriptItem],
5385    ) -> anyhow::Result<()> {
5386        self.emit(RoderEvent::ReliabilityLimitRecorded(
5387            ReliabilityLimitRecorded {
5388                context: ReliabilityContext {
5389                    thread_id: thread_id.clone(),
5390                    turn_id: turn_id.clone(),
5391                    tool_id: None,
5392                    tool_name: None,
5393                    provider: Some(provider.to_string()),
5394                    model: Some(model.to_string()),
5395                },
5396                error_class: limit.error_class,
5397                limit_kind: limit.limit_kind,
5398                decision: limit.decision,
5399                current: limit.current,
5400                limit: limit.limit,
5401                details: ReliabilityDetails::redacted(&limit.message),
5402                timestamp: OffsetDateTime::now_utc(),
5403            },
5404        ))
5405        .await;
5406        let partial_result = turn_partial_result(transcript);
5407        self.emit(RoderEvent::TurnPartialResult(TurnPartialResult {
5408            thread_id: thread_id.clone(),
5409            turn_id: turn_id.clone(),
5410            summary: partial_result.clone(),
5411            timestamp: OffsetDateTime::now_utc(),
5412        }))
5413        .await;
5414        let message = format!("reliability limit reached: {}", limit.message);
5415        self.persist_turn_item(
5416            thread_id,
5417            turn_id,
5418            &TranscriptItem::Error(ErrorRecord {
5419                message: format!("{message}: {partial_result}"),
5420            }),
5421        )
5422        .await?;
5423        self.emit(RoderEvent::TurnFailed(TurnFailed {
5424            thread_id: thread_id.clone(),
5425            turn_id: turn_id.clone(),
5426            error: message.clone(),
5427            error_kind: Some("reliability_limit".to_string()),
5428            usage: None,
5429            timestamp: OffsetDateTime::now_utc(),
5430        }))
5431        .await;
5432        self.complete_team_member_turn_with_result(
5433            thread_id,
5434            turn_id,
5435            TeamMemberStatus::Failed,
5436            None,
5437            Some(format!("{message}: {partial_result}")),
5438        )
5439        .await?;
5440        Ok(())
5441    }
5442
5443    async fn append_steers(
5444        &self,
5445        req: &StartTurnRequest,
5446        turn_id: &TurnId,
5447        transcript: &mut Vec<TranscriptItem>,
5448        steers: Vec<QueuedTurnSteer>,
5449    ) -> anyhow::Result<()> {
5450        for queued in steers {
5451            let mut steer = queued.message;
5452            steer.text = steer.text.trim().to_string();
5453            if steer.text.is_empty() && steer.images.is_empty() {
5454                continue;
5455            }
5456            let item = TranscriptItem::UserMessage(steer);
5457            self.persist_turn_item(&req.thread_id, turn_id, &item)
5458                .await?;
5459            transcript.push(item);
5460            if let Some(ack) = queued.mailbox_ack {
5461                self.teams
5462                    .mark_mailbox_messages_delivered(&ack.team_id, turn_id, &ack.message_ids)
5463                    .await?;
5464            }
5465        }
5466        Ok(())
5467    }
5468
5469    async fn persist_model_profile_segment(
5470        &self,
5471        thread_id: &ThreadId,
5472        turn_id: &TurnId,
5473        profile: Option<&ModelHarnessProfile>,
5474        provider: &str,
5475        model: &str,
5476        segment: &str,
5477    ) -> anyhow::Result<()> {
5478        let item = TranscriptItem::ProviderMetadata(model_profile_segment_metadata(
5479            profile, provider, model, segment,
5480        ));
5481        self.persist_turn_item(thread_id, turn_id, &item).await
5482    }
5483
5484    /**
5485     * Allowlists apply to built-in tools only; external tools are advertised with their
5486     * host-supplied schemas as given. An external tool shadows a built-in with the same name in
5487     * both advertisement and dispatch (see `route_tool_call`).
5488     */
5489    fn filtered_tool_specs(
5490        &self,
5491        cfg: &RuntimeConfig,
5492        model: &str,
5493        profile: Option<&ModelHarnessProfile>,
5494        thread_allowlist: &[String],
5495        external_tools: &[roder_api::tools::ToolSpec],
5496    ) -> Vec<roder_api::tools::ToolSpec> {
5497        let mut specs = self
5498            .tool_registry
5499            .specs_for_edit_tool_with_schema_policy(
5500                edit_tool_for_model(cfg, model),
5501                schema_policy_for_model(profile),
5502            )
5503            .into_iter()
5504            .filter(|spec| {
5505                allowlist_permits(&cfg.tool_allowlist, &spec.name)
5506                    && allowlist_permits(thread_allowlist, &spec.name)
5507                    && !external_tools.iter().any(|tool| tool.name == spec.name)
5508            })
5509            .collect::<Vec<_>>();
5510        specs.extend(external_tools.iter().cloned());
5511        specs
5512    }
5513
5514    fn task_ledger_tool_specs(
5515        &self,
5516        profile: Option<&ModelHarnessProfile>,
5517    ) -> Vec<roder_api::tools::ToolSpec> {
5518        self.tool_registry
5519            .get(TASK_LEDGER_TOOL_NAME)
5520            .map(|tool| {
5521                tool.spec()
5522                    .normalized_for_model_profile(schema_policy_for_model(profile))
5523            })
5524            .into_iter()
5525            .collect()
5526    }
5527
5528    pub(crate) fn engine_for(&self, provider: &str) -> anyhow::Result<Arc<dyn InferenceEngine>> {
5529        self.registry
5530            .inference_engine(provider)
5531            .or_else(|| {
5532                self.registry
5533                    .default_inference_engine()
5534                    .filter(|engine| provider.is_empty() || engine.id() == provider)
5535            })
5536            .ok_or_else(|| anyhow::anyhow!("inference provider {provider:?} is not registered"))
5537    }
5538
5539    pub async fn emit(&self, event: RoderEvent) -> EventEnvelope {
5540        if let Some(record) = Self::lifecycle_record_for_event(&event) {
5541            let _ = self.persist_turn_lifecycle_record(&record).await;
5542        }
5543        let envelope = self.bus.emit(event);
5544        if let (Some(store), Some(thread_id)) = (&self.thread_store, envelope.thread_id.as_ref())
5545            && should_persist_thread_event(thread_id)
5546        {
5547            let _ = store.append_event(thread_id, &envelope).await;
5548        }
5549        // Registered event sinks (e.g. process extensions) receive the
5550        // persisted envelope through bounded per-sink queues; a slow sink
5551        // never blocks emit or turn progress.
5552        let dispatcher = self
5553            .event_sink_dispatcher
5554            .get_or_init(|| async {
5555                crate::event_sink_dispatch::EventSinkDispatcher::start(
5556                    &self.registry.event_sinks,
5557                    self.bus.clone(),
5558                )
5559            })
5560            .await;
5561        if !dispatcher.is_empty() {
5562            dispatcher.dispatch(&envelope, &self.bus);
5563        }
5564        envelope
5565    }
5566
5567    /// Records a projected thread item event and persists it through the configured thread store.
5568    ///
5569    /// App-server protocol notification bridges currently call this after translating runtime
5570    /// events into item events. Headless runtime consumers that subscribe to `RoderEvent` directly
5571    /// must make the same call if they need the derived item event stream persisted.
5572    pub async fn record_thread_item_event_kind(
5573        &self,
5574        thread_id: &ThreadId,
5575        turn_id: &TurnId,
5576        timestamp: OffsetDateTime,
5577        kind: ThreadItemEventKind,
5578    ) -> anyhow::Result<ThreadItemEvent> {
5579        let seq = self.next_thread_item_event_seq(thread_id).await?;
5580        let item_event = ThreadItemEvent {
5581            seq,
5582            event_id: format!("{turn_id}-item-event-{seq}"),
5583            thread_id: thread_id.clone(),
5584            turn_id: turn_id.clone(),
5585            timestamp,
5586            event: kind,
5587        };
5588        if let Some(store) = &self.thread_store {
5589            store.append_item_event(thread_id, &item_event).await?;
5590        }
5591        self.remember_thread_item_event(&item_event).await?;
5592        Ok(item_event)
5593    }
5594
5595    async fn next_thread_item_event_seq(&self, thread_id: &ThreadId) -> anyhow::Result<u64> {
5596        self.ensure_thread_item_cache(thread_id).await?;
5597        Ok(self
5598            .thread_item_cache
5599            .lock()
5600            .await
5601            .next_item_event_seq(thread_id))
5602    }
5603
5604    pub async fn thread_item_exists(
5605        &self,
5606        thread_id: &ThreadId,
5607        turn_id: &TurnId,
5608        item_id: &str,
5609    ) -> anyhow::Result<bool> {
5610        self.ensure_thread_item_cache(thread_id).await?;
5611        Ok(self
5612            .thread_item_cache
5613            .lock()
5614            .await
5615            .thread_item_exists(thread_id, turn_id, item_id))
5616    }
5617
5618    pub async fn current_reasoning_item_id(
5619        &self,
5620        thread_id: &ThreadId,
5621        turn_id: &TurnId,
5622    ) -> anyhow::Result<Option<String>> {
5623        self.ensure_thread_item_cache(thread_id).await?;
5624        Ok(self
5625            .thread_item_cache
5626            .lock()
5627            .await
5628            .current_reasoning_item_id(thread_id, turn_id))
5629    }
5630
5631    async fn remember_thread_item_event(&self, item_event: &ThreadItemEvent) -> anyhow::Result<()> {
5632        self.ensure_thread_item_cache(&item_event.thread_id).await?;
5633        self.thread_item_cache
5634            .lock()
5635            .await
5636            .remember_item_event(item_event);
5637        Ok(())
5638    }
5639
5640    pub async fn latest_transcript_item_index(
5641        &self,
5642        thread_id: &ThreadId,
5643        turn_id: &TurnId,
5644    ) -> anyhow::Result<Option<usize>> {
5645        self.ensure_thread_item_cache(thread_id).await?;
5646        Ok(self
5647            .thread_item_cache
5648            .lock()
5649            .await
5650            .latest_transcript_item_index(thread_id, turn_id))
5651    }
5652
5653    async fn next_transcript_item_index(
5654        &self,
5655        thread_id: &ThreadId,
5656        turn_id: &TurnId,
5657    ) -> anyhow::Result<usize> {
5658        self.ensure_thread_item_cache(thread_id).await?;
5659        Ok(self
5660            .thread_item_cache
5661            .lock()
5662            .await
5663            .next_transcript_item_index(thread_id, turn_id))
5664    }
5665
5666    async fn remember_transcript_item_index(
5667        &self,
5668        thread_id: &ThreadId,
5669        turn_id: &TurnId,
5670        item_index: usize,
5671    ) -> anyhow::Result<()> {
5672        self.ensure_thread_item_cache(thread_id).await?;
5673        self.thread_item_cache
5674            .lock()
5675            .await
5676            .remember_transcript_item_index(thread_id, turn_id, item_index);
5677        Ok(())
5678    }
5679
5680    async fn ensure_thread_item_cache(&self, thread_id: &ThreadId) -> anyhow::Result<()> {
5681        if self
5682            .thread_item_cache
5683            .lock()
5684            .await
5685            .contains_thread(thread_id)
5686        {
5687            return Ok(());
5688        }
5689
5690        let snapshot = if let Some(store) = &self.thread_store {
5691            store.load_thread(thread_id).await?
5692        } else {
5693            None
5694        };
5695        self.thread_item_cache.lock().await.ensure_thread(
5696            thread_id,
5697            ThreadItemCacheEntry::from_snapshot(snapshot.as_ref()),
5698        );
5699        Ok(())
5700    }
5701
5702    pub(crate) async fn persist_turn_item(
5703        &self,
5704        thread_id: &ThreadId,
5705        turn_id: &TurnId,
5706        item: &TranscriptItem,
5707    ) -> anyhow::Result<()> {
5708        let item_index = self.next_transcript_item_index(thread_id, turn_id).await?;
5709        let timestamp = OffsetDateTime::now_utc();
5710        self.emit(RoderEvent::TranscriptItemAppended(TranscriptItemAppended {
5711            thread_id: thread_id.clone(),
5712            turn_id: turn_id.clone(),
5713            item_type: match item {
5714                TranscriptItem::UserMessage(_) => "user_message",
5715                TranscriptItem::AssistantMessage(_) => "assistant_message",
5716                TranscriptItem::ReasoningSummary(_) => "reasoning_summary",
5717                TranscriptItem::ToolCall(_) => "tool_call",
5718                TranscriptItem::ToolResult(_) => "tool_result",
5719                TranscriptItem::FileChange(_) => "file_change",
5720                TranscriptItem::ContextCompaction(_) => "context_compaction",
5721                TranscriptItem::Error(_) => "error",
5722                TranscriptItem::ProviderMetadata(_) => "provider_metadata",
5723            }
5724            .to_string(),
5725            item_index: Some(item_index),
5726            item: Some(item.clone()),
5727            timestamp,
5728        }))
5729        .await;
5730        self.remember_transcript_item_index(thread_id, turn_id, item_index)
5731            .await?;
5732        Ok(())
5733    }
5734}
5735
5736fn transcript_has_images(transcript: &[TranscriptItem]) -> bool {
5737    transcript.iter().any(|item| {
5738        matches!(
5739            item,
5740            TranscriptItem::UserMessage(message) if !message.images.is_empty()
5741        )
5742    })
5743}
5744
5745fn transcript_has_task_ledger(transcript: &[TranscriptItem]) -> bool {
5746    transcript.iter().any(|item| {
5747        matches!(
5748            item,
5749            TranscriptItem::ToolResult(result)
5750                if result.name.as_deref() == Some(TASK_LEDGER_TOOL_NAME) && !result.is_error
5751        )
5752    })
5753}
5754
5755fn task_ledger_completion_prompt(transcript: &[TranscriptItem]) -> Option<String> {
5756    let latest = transcript.iter().rev().find_map(|item| match item {
5757        TranscriptItem::ToolResult(result)
5758            if result.name.as_deref() == Some(TASK_LEDGER_TOOL_NAME) && !result.is_error =>
5759        {
5760            Some(result.result.as_str())
5761        }
5762        _ => None,
5763    })?;
5764    if !task_ledger_has_open_items(latest) {
5765        return None;
5766    }
5767
5768    let mut ledger = latest.chars().take(1500).collect::<String>();
5769    if latest.chars().nth(1500).is_some() {
5770        ledger.push_str("...");
5771    }
5772    Some(format!(
5773        "Task Ledger Completion Required: the latest task ledger still has pending or in-progress items. Do not provide a final answer yet. Use tools to complete the remaining scoreable work, create or update any required output files, then call `{TASK_LEDGER_TOOL_NAME}` with every task completed and evidence before finalizing.\n\nLatest ledger:\n{ledger}"
5774    ))
5775}
5776
5777fn task_ledger_deadline_completion_prompt(
5778    remaining_seconds: u64,
5779    reserve_seconds: u64,
5780    completion_prompt: &str,
5781) -> String {
5782    format!(
5783        "Eval deadline scoreable completion: {remaining_seconds} seconds remain in the {reserve_seconds}-second finalization reserve. Do not browse, search, or start slow work. Use the available tools now to create or update the required scoreable output files, run only a quick local check if needed, then update the task ledger to completed before finalizing.\n\n{completion_prompt}"
5784    )
5785}
5786
5787fn task_ledger_scoreable_checkpoint_prompt(
5788    remaining_seconds: u64,
5789    completion_prompt: &str,
5790) -> String {
5791    format!(
5792        "Scoreable Output Checkpoint: {remaining_seconds} seconds remain before the eval deadline. Before any further research, browsing, or long commands, use tools now to ensure the required output file(s) exist with the best evidence-backed answer, even if provisional. If a scoreable file already exists, read it and preserve that candidate unless you have stronger task-specific evidence for a replacement. Do not overwrite a plausible dated, historical, or local-evidence candidate with a current live-page, partial-coverage, or weaker guess merely to refresh the checkpoint. You may continue refining afterward, but do not apologize or finalize until the scoreable file exists and the task ledger is updated.\n\n{completion_prompt}"
5793    )
5794}
5795
5796fn task_ledger_has_open_items(ledger: &str) -> bool {
5797    ledger.lines().any(|line| {
5798        let line = line.trim_start();
5799        line.starts_with("- pending:") || line.starts_with("- in_progress:")
5800    })
5801}
5802
5803fn turn_deadline_for_config(cfg: &RuntimeConfig) -> Option<OffsetDateTime> {
5804    cfg.turn_deadline_seconds
5805        .filter(|seconds| *seconds > 0)
5806        .map(|seconds| OffsetDateTime::now_utc() + Duration::seconds(seconds as i64))
5807}
5808
5809fn deadline_expired(deadline: OffsetDateTime) -> bool {
5810    OffsetDateTime::now_utc() >= deadline
5811}
5812
5813pub(crate) fn deadline_remaining_seconds(deadline: Option<OffsetDateTime>) -> Option<u64> {
5814    let deadline = deadline?;
5815    if deadline <= OffsetDateTime::now_utc() {
5816        return Some(0);
5817    }
5818    Some(
5819        (deadline - OffsetDateTime::now_utc())
5820            .unsigned_abs()
5821            .as_secs()
5822            .max(1),
5823    )
5824}
5825
5826fn deadline_instant(deadline: OffsetDateTime) -> tokio::time::Instant {
5827    let now = OffsetDateTime::now_utc();
5828    if deadline <= now {
5829        return tokio::time::Instant::now();
5830    }
5831    tokio::time::Instant::now() + (deadline - now).unsigned_abs()
5832}
5833
5834fn inference_timeout_deadline(
5835    deadline: Option<OffsetDateTime>,
5836    runtime_profile: RuntimeProfile,
5837    task_ledger_required: bool,
5838    reserve_seconds: u64,
5839    finalization_requested: bool,
5840    task_ledger_scoreable_checkpoints: u8,
5841    transcript: &[TranscriptItem],
5842) -> Option<(OffsetDateTime, InferenceTimeoutAction)> {
5843    let deadline = deadline?;
5844    if runtime_profile == RuntimeProfile::Eval
5845        && task_ledger_required
5846        && !finalization_requested
5847        && task_ledger_scoreable_checkpoints < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
5848        && TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS > reserve_seconds
5849        && task_ledger_completion_prompt(transcript).is_some()
5850    {
5851        let checkpoint_deadline =
5852            deadline - Duration::seconds(TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS as i64);
5853        if checkpoint_deadline > OffsetDateTime::now_utc() {
5854            return Some((
5855                checkpoint_deadline,
5856                InferenceTimeoutAction::ScoreableCheckpoint,
5857            ));
5858        }
5859    }
5860    if runtime_profile == RuntimeProfile::Eval && !finalization_requested {
5861        return Some((
5862            deadline - Duration::seconds(reserve_seconds as i64),
5863            InferenceTimeoutAction::Finalization,
5864        ));
5865    }
5866    Some((deadline, InferenceTimeoutAction::Finalization))
5867}
5868
5869fn turn_partial_result(transcript: &[TranscriptItem]) -> String {
5870    let tool_results = transcript
5871        .iter()
5872        .filter(|item| matches!(item, TranscriptItem::ToolResult(_)))
5873        .count();
5874    let assistant_messages = transcript
5875        .iter()
5876        .filter(|item| matches!(item, TranscriptItem::AssistantMessage(_)))
5877        .count();
5878    format!(
5879        "partial turn state: {} transcript items, {assistant_messages} assistant messages, {tool_results} tool results",
5880        transcript.len()
5881    )
5882}
5883
5884fn reasoning_for_model(cfg: &RuntimeConfig, model: &str) -> ReasoningConfig {
5885    let level = effective_reasoning_for_model(cfg, model);
5886    match level.as_str() {
5887        "" | REASONING_NONE => ReasoningConfig::default(),
5888        level => ReasoningConfig {
5889            enabled: true,
5890            level: Some(level.to_string()),
5891        },
5892    }
5893}
5894
5895fn server_side_compaction_threshold(cfg: &RuntimeConfig, model: &str) -> Option<u32> {
5896    let entry = lookup_model(model)?;
5897    if !entry.supports_compaction {
5898        return None;
5899    }
5900    cfg.auto_compact_token_limit
5901        .or_else(|| {
5902            model_profile_for_model(cfg, model).and_then(|profile| profile.auto_compact_token_limit)
5903        })
5904        .or(Some(entry.auto_compact_token_limit))
5905        .filter(|threshold| *threshold > 0)
5906}
5907
5908pub(crate) fn tool_search_for_provider_model(
5909    cfg: &RuntimeConfig,
5910    provider: &str,
5911    model: &str,
5912) -> ToolSearchConfig {
5913    let mut resolved = cfg.tool_search.clone();
5914    if let Some(provider_config) = cfg.provider_tool_search.get(provider) {
5915        provider_config.apply_to(&mut resolved);
5916    }
5917    if let Some(model_config) = cfg.model_tool_search.get(model) {
5918        model_config.apply_to(&mut resolved);
5919    }
5920    resolved
5921}
5922
5923fn parallel_tool_calls_for_model(cfg: &RuntimeConfig, model: &str) -> bool {
5924    cfg.model_parallel_tool_calls
5925        .get(model)
5926        .copied()
5927        .or_else(|| {
5928            model_profile_for_model(cfg, model).and_then(|profile| profile.parallel_tool_calls)
5929        })
5930        .unwrap_or(true)
5931}
5932
5933/// Count the children an `agent_swarm` call will launch from its arguments
5934/// (item-based spawns plus resumes). Lenient: returns 0 on malformed input.
5935fn agent_swarm_child_count(arguments: &str) -> usize {
5936    serde_json::from_str::<roder_api::subagents::AgentSwarmRequest>(arguments)
5937        .map(|request| request.items.len() + request.resume_agent_ids.len())
5938        .unwrap_or(0)
5939}
5940
5941/// Parse `completed`/`failed`/`aborted` counts from an `<agent_swarm_result>`
5942/// summary line, e.g. `<summary>completed: 2, failed: 1</summary>`. Omitted
5943/// buckets default to 0. Returns `None` when the text is not a swarm result.
5944fn parse_swarm_counts(text: &str) -> Option<(usize, usize, usize)> {
5945    if !text.contains("<agent_swarm_result>") {
5946        return None;
5947    }
5948    let bucket = |label: &str| -> usize {
5949        let needle = format!("{label}: ");
5950        text.find(&needle)
5951            .map(|start| start + needle.len())
5952            .map(|start| {
5953                text[start..]
5954                    .chars()
5955                    .take_while(|c| c.is_ascii_digit())
5956                    .collect::<String>()
5957            })
5958            .and_then(|digits| digits.parse().ok())
5959            .unwrap_or(0)
5960    };
5961    Some((bucket("completed"), bucket("failed"), bucket("aborted")))
5962}
5963
5964fn effective_reasoning_for_model(cfg: &RuntimeConfig, model: &str) -> String {
5965    let base_reasoning = default_effective_reasoning_for_model(cfg, model);
5966    if cfg.dynamic_workflows.effort_profile == DynamicWorkflowEffortProfile::Ultracode {
5967        return ultracode_reasoning_level_for_model(
5968            model,
5969            &cfg.speed_policy.ultracode_reasoning,
5970            &base_reasoning,
5971        );
5972    }
5973    base_reasoning
5974}
5975
5976fn default_effective_reasoning_for_model(cfg: &RuntimeConfig, model: &str) -> String {
5977    let Some(entry) = lookup_model(model) else {
5978        return cfg
5979            .reasoning
5980            .clone()
5981            .unwrap_or_else(|| REASONING_NONE.to_string());
5982    };
5983    if entry.supported_reasoning.is_empty() {
5984        return REASONING_NONE.to_string();
5985    }
5986    cfg.reasoning
5987        .as_deref()
5988        .filter(|reasoning| {
5989            entry
5990                .supported_reasoning
5991                .iter()
5992                .any(|option| option.effort == *reasoning)
5993        })
5994        .map(str::to_string)
5995        .or_else(|| {
5996            model_profile_for_model(cfg, model)
5997                .and_then(|profile| profile.reasoning.orientation)
5998                .filter(|reasoning| {
5999                    entry
6000                        .supported_reasoning
6001                        .iter()
6002                        .any(|option| option.effort == reasoning)
6003                })
6004        })
6005        .unwrap_or_else(|| entry.default_reasoning.to_string())
6006}
6007
6008fn validate_reasoning_effort(model: &str, effort: &str) -> anyhow::Result<()> {
6009    if effort == REASONING_NONE && !model_supports_reasoning(model, effort) {
6010        return Ok(());
6011    }
6012    let Some(entry) = lookup_model(model) else {
6013        return Ok(());
6014    };
6015    if entry
6016        .supported_reasoning
6017        .iter()
6018        .any(|option| option.effort == effort)
6019    {
6020        Ok(())
6021    } else {
6022        anyhow::bail!("model {model} does not support reasoning effort {effort}")
6023    }
6024}
6025
6026fn validate_runtime_config_reasoning(cfg: &RuntimeConfig) -> anyhow::Result<()> {
6027    let Some(reasoning) = cfg.reasoning.as_deref() else {
6028        return Ok(());
6029    };
6030    let Some(entry) = lookup_model(&cfg.default_model) else {
6031        return Ok(());
6032    };
6033    if entry.provider != PROVIDER_GEMINI {
6034        return Ok(());
6035    }
6036    validate_reasoning_effort(&cfg.default_model, reasoning)
6037}
6038
6039fn validate_runtime_inference_router_config(
6040    registry: &ExtensionRegistry,
6041    cfg: &RuntimeConfig,
6042) -> anyhow::Result<()> {
6043    if !cfg.inference_router.enabled {
6044        return Ok(());
6045    }
6046    let Some(router_id) = cfg.inference_router.router_id.as_deref() else {
6047        anyhow::bail!("inference_router.enabled requires inference_router.router");
6048    };
6049    if registry.inference_router(router_id).is_some() {
6050        return Ok(());
6051    }
6052    let available = registry
6053        .inference_routers
6054        .iter()
6055        .map(|router| router.id())
6056        .collect::<Vec<_>>()
6057        .join(", ");
6058    if available.is_empty() {
6059        anyhow::bail!("inference router {router_id:?} is not registered");
6060    }
6061    anyhow::bail!(
6062        "inference router {router_id:?} is not registered; available routers: {available}"
6063    );
6064}
6065
6066fn model_supports_reasoning(model: &str, effort: &str) -> bool {
6067    lookup_model(model)
6068        .map(|entry| {
6069            entry
6070                .supported_reasoning
6071                .iter()
6072                .any(|option| option.effort == effort)
6073        })
6074        .unwrap_or(false)
6075}
6076
6077fn is_final_answer_phase(phase: Option<&str>) -> bool {
6078    phase.is_none_or(|phase| phase.is_empty() || phase == FINAL_ANSWER_PHASE)
6079}
6080
6081fn edit_tool_for_model<'a>(cfg: &'a RuntimeConfig, model: &'a str) -> Option<&'a str> {
6082    cfg.model_edit_tools
6083        .get(model)
6084        .map(String::as_str)
6085        .or_else(|| {
6086            cfg.model_profiles
6087                .get(model)
6088                .and_then(|profile| profile.edit_tool.as_deref())
6089        })
6090        .or_else(|| lookup_model(model).and_then(|entry| entry.edit_tool))
6091        .or(Some(EDIT_TOOL_EDIT))
6092}
6093
6094fn model_profile_for_model(cfg: &RuntimeConfig, model: &str) -> Option<ModelHarnessProfile> {
6095    cfg.model_profiles
6096        .get(model)
6097        .cloned()
6098        .or_else(|| built_in_model_profile(model))
6099}
6100
6101pub(crate) fn allowlist_permits(allowlist: &[String], tool_name: &str) -> bool {
6102    allowlist.is_empty() || allowlist.iter().any(|allowed| allowed == tool_name)
6103}
6104
6105/// Provider-aware profile resolution for the active turn.
6106///
6107/// Many model ids are shared across providers (for example Cursor proxies
6108/// `claude-opus-4-8`). Resolving the harness profile by id alone picks the
6109/// first catalog entry, which assigns cross-provider ids the wrong family and
6110/// instruction overlay (e.g. a `cursor/claude-opus-4-8` turn would otherwise
6111/// inherit the Anthropic overlay). Prefer the explicit `(provider, model)`
6112/// catalog entry, keeping user-configured profile overrides as the top
6113/// precedence.
6114fn model_profile_for_provider_model(
6115    cfg: &RuntimeConfig,
6116    provider: &str,
6117    model: &str,
6118) -> Option<ModelHarnessProfile> {
6119    cfg.model_profiles
6120        .get(model)
6121        .cloned()
6122        .or_else(|| built_in_model_profile_for_provider(provider, model))
6123}
6124
6125fn schema_policy_for_model(profile: Option<&ModelHarnessProfile>) -> ModelSchemaPolicy {
6126    profile
6127        .map(|profile| profile.schema_policy)
6128        .unwrap_or_default()
6129}
6130
6131fn model_profile_segment_metadata(
6132    profile: Option<&ModelHarnessProfile>,
6133    provider: &str,
6134    model: &str,
6135    segment: &str,
6136) -> serde_json::Value {
6137    serde_json::json!({
6138        "kind": MODEL_PROFILE_TRACE_KIND,
6139        "segment": segment,
6140        "provider": provider,
6141        "model": model,
6142        "profileModel": profile.map(|profile| profile.model.as_str()).unwrap_or(model),
6143        "providerFamily": profile.map(|profile| profile.provider_family),
6144        "editTool": profile.and_then(|profile| profile.edit_tool.as_deref()),
6145        "schemaPolicy": profile.map(|profile| profile.schema_policy),
6146        "instructionOverlay": profile.map(|profile| profile.instruction_overlay),
6147        "parallelToolCalls": profile.and_then(|profile| profile.parallel_tool_calls),
6148        "autoCompactTokenLimit": profile.and_then(|profile| profile.auto_compact_token_limit),
6149    })
6150}
6151
6152fn model_switch_summary(
6153    transcript: &[TranscriptItem],
6154    profile: Option<&ModelHarnessProfile>,
6155    provider: &str,
6156    model: &str,
6157    tools: &[roder_api::tools::ToolSpec],
6158) -> Option<String> {
6159    let previous = latest_model_profile_segment(transcript)?;
6160    let previous_model = previous
6161        .get("model")
6162        .and_then(serde_json::Value::as_str)
6163        .unwrap_or_default();
6164    let previous_provider = previous
6165        .get("provider")
6166        .and_then(serde_json::Value::as_str)
6167        .unwrap_or_default();
6168    if previous_model == model && previous_provider == provider {
6169        return None;
6170    }
6171
6172    let previous_profile = previous
6173        .get("profileModel")
6174        .and_then(serde_json::Value::as_str)
6175        .unwrap_or(previous_model);
6176    let current_profile = profile
6177        .map(|profile| profile.model.as_str())
6178        .unwrap_or(model);
6179    let previous_edit_tool = previous
6180        .get("editTool")
6181        .and_then(serde_json::Value::as_str)
6182        .unwrap_or("none");
6183    let current_edit_tool = profile
6184        .and_then(|profile| profile.edit_tool.as_deref())
6185        .unwrap_or("none");
6186    let tool_names = tools
6187        .iter()
6188        .map(|tool| tool.name.as_str())
6189        .take(12)
6190        .collect::<Vec<_>>()
6191        .join(", ");
6192    Some(format!(
6193        "{MODEL_SWITCH_SUMMARY_PREFIX} previous profile {previous_provider}/{previous_profile} used edit tool {previous_edit_tool}. Current profile {provider}/{current_profile} uses edit tool {current_edit_tool}. Available tools now: {}.",
6194        if tool_names.is_empty() {
6195            "none"
6196        } else {
6197            &tool_names
6198        }
6199    ))
6200}
6201
6202fn latest_model_profile_segment(transcript: &[TranscriptItem]) -> Option<&serde_json::Value> {
6203    transcript.iter().rev().find_map(|item| {
6204        let TranscriptItem::ProviderMetadata(value) = item else {
6205            return None;
6206        };
6207        (value.get("kind").and_then(serde_json::Value::as_str) == Some(MODEL_PROFILE_TRACE_KIND))
6208            .then_some(value)
6209    })
6210}
6211
6212pub fn validate_edit_tool(value: &str) -> anyhow::Result<()> {
6213    match value.trim() {
6214        EDIT_TOOL_PATCH | EDIT_TOOL_EDIT => Ok(()),
6215        _ => anyhow::bail!(
6216            "unsupported edit_tool {value:?}; allowed values: {EDIT_TOOL_PATCH}, {EDIT_TOOL_EDIT}"
6217        ),
6218    }
6219}
6220
6221fn should_persist_thread_event(thread_id: &str) -> bool {
6222    !is_synthetic_event_thread_id(thread_id)
6223}
6224
6225#[cfg(test)]
6226#[path = "runtime/codex_v2_tests.rs"]
6227mod codex_v2_tests;
6228
6229#[cfg(test)]
6230#[path = "runtime/codex_v2_lifecycle_tests.rs"]
6231mod codex_v2_lifecycle_tests;
6232
6233#[cfg(test)]
6234mod tests {
6235    use super::*;
6236    use futures::stream;
6237    use roder_api::catalog::{
6238        PROVIDER_MOCK, REASONING_HIGH, REASONING_LOW, REASONING_MEDIUM, REASONING_MINIMAL,
6239        REASONING_NONE, REASONING_XHIGH,
6240    };
6241    use roder_api::extension::ExtensionRegistryBuilder;
6242    use roder_api::inference::{
6243        CompletionMetadata, InferenceCapabilities, InferenceEngine, InferenceEventStream,
6244        InferenceProviderContext, InferenceTurnContext, MessageDelta, ModelDescriptor,
6245        ModelInstructionOverlay, ModelProfileReasoning, ModelSchemaPolicy, ProviderFamily,
6246        ReasoningEffortDescriptor,
6247    };
6248    use roder_api::inference_routing::{
6249        InferenceRouter, InferenceRoutingContext, InferenceRoutingDecision, InferenceRoutingOutcome,
6250    };
6251    use roder_api::thread::ThreadStoreFactory;
6252    use roder_api::tools::{ToolContributor, ToolExecutor, ToolSpec};
6253    use roder_ext_jsonl_thread_store::store::JsonlThreadStoreFactory;
6254    use std::sync::Mutex as StdMutex;
6255
6256    fn test_workspace() -> String {
6257        std::env::current_dir().unwrap().display().to_string()
6258    }
6259
6260    #[test]
6261    fn interactive_explicit_turn_deadline_has_bounded_remaining_time() {
6262        let config = RuntimeConfig {
6263            runtime_profile: RuntimeProfile::Interactive,
6264            turn_deadline_seconds: Some(60),
6265            ..RuntimeConfig::default()
6266        };
6267
6268        let deadline = turn_deadline_for_config(&config).expect("configured turn deadline");
6269        let remaining = deadline_remaining_seconds(Some(deadline));
6270
6271        assert!(
6272            matches!(remaining, Some(1..=60)),
6273            "interactive deadline must be positive and no longer than configured: {remaining:?}"
6274        );
6275    }
6276
6277    #[test]
6278    fn interactive_without_turn_deadline_stays_unbounded() {
6279        let config = RuntimeConfig {
6280            runtime_profile: RuntimeProfile::Interactive,
6281            turn_deadline_seconds: None,
6282            ..RuntimeConfig::default()
6283        };
6284
6285        assert_eq!(turn_deadline_for_config(&config), None);
6286    }
6287
6288    struct MetadataMissingStore;
6289
6290    #[async_trait::async_trait]
6291    impl ThreadStore for MetadataMissingStore {
6292        fn id(&self) -> roder_api::thread::ThreadStoreId {
6293            "metadata-missing-store".to_string()
6294        }
6295
6296        async fn create_thread(&self, metadata: ThreadMetadata) -> anyhow::Result<ThreadMetadata> {
6297            Ok(metadata)
6298        }
6299
6300        async fn list_threads(&self) -> anyhow::Result<Vec<ThreadMetadata>> {
6301            Ok(Vec::new())
6302        }
6303
6304        async fn load_thread(
6305            &self,
6306            _thread_id: &ThreadId,
6307        ) -> anyhow::Result<Option<ThreadSnapshot>> {
6308            Ok(Some(ThreadSnapshot {
6309                metadata: None,
6310                ..ThreadSnapshot::default()
6311            }))
6312        }
6313
6314        async fn append_event(
6315            &self,
6316            _thread_id: &ThreadId,
6317            _envelope: &EventEnvelope,
6318        ) -> anyhow::Result<()> {
6319            Ok(())
6320        }
6321    }
6322
6323    struct MetadataMissingStoreFactory;
6324
6325    impl ThreadStoreFactory for MetadataMissingStoreFactory {
6326        fn id(&self) -> roder_api::thread::ThreadStoreId {
6327            "metadata-missing-store".to_string()
6328        }
6329
6330        fn create(&self) -> Arc<dyn ThreadStore> {
6331            Arc::new(MetadataMissingStore)
6332        }
6333    }
6334
6335    #[test]
6336    fn synthetic_app_server_events_are_not_thread_events() {
6337        for thread_id in ["app-server", "runtime", "thread-workflow"] {
6338            assert!(!should_persist_thread_event(thread_id));
6339        }
6340        assert!(should_persist_thread_event("thread-discovery"));
6341        assert!(should_persist_thread_event("thread-plan"));
6342        assert!(should_persist_thread_event("thread-process"));
6343        assert!(should_persist_thread_event("thread-1"));
6344    }
6345
6346    #[test]
6347    fn server_side_compaction_uses_catalog_ninety_percent_default() {
6348        assert_eq!(
6349            server_side_compaction_threshold(&RuntimeConfig::default(), "gpt-5.5"),
6350            Some(945_000)
6351        );
6352        assert_eq!(
6353            server_side_compaction_threshold(&RuntimeConfig::default(), "gpt-5.3-codex-spark"),
6354            Some(115_200)
6355        );
6356    }
6357
6358    #[test]
6359    fn server_side_compaction_respects_explicit_config_override() {
6360        let cfg = RuntimeConfig {
6361            auto_compact_token_limit: Some(123_456),
6362            ..RuntimeConfig::default()
6363        };
6364
6365        assert_eq!(
6366            server_side_compaction_threshold(&cfg, "gpt-5.5"),
6367            Some(123_456)
6368        );
6369    }
6370
6371    #[tokio::test]
6372    async fn pre_request_compaction_runs_when_server_side_model_is_at_context_window() {
6373        let captured = Arc::new(StdMutex::new(None));
6374        let mut builder = ExtensionRegistryBuilder::new();
6375        builder.inference_engine(Arc::new(CapturingEngine {
6376            request: captured.clone(),
6377        }));
6378        let thread_root = std::env::temp_dir().join(format!(
6379            "roder-pre-request-compaction-{}",
6380            uuid::Uuid::new_v4()
6381        ));
6382        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
6383            base_path: thread_root.clone(),
6384        }));
6385        let runtime = Arc::new(
6386            Runtime::new(
6387                builder.build().unwrap(),
6388                RuntimeConfig {
6389                    default_provider: PROVIDER_MOCK.to_string(),
6390                    default_model: "gpt-5.5".to_string(),
6391                    file_backed_dynamic_context: true,
6392                    ..RuntimeConfig::default()
6393                },
6394            )
6395            .unwrap(),
6396        );
6397        let thread_id = runtime
6398            .create_thread(Some("Pre-request compaction".to_string()))
6399            .await
6400            .unwrap()
6401            .thread_id;
6402        let old_turn = "old-turn".to_string();
6403        runtime
6404            .persist_turn_item(
6405                &thread_id,
6406                &old_turn,
6407                &TranscriptItem::UserMessage(UserMessage::text("old context ".repeat(4_300_000))),
6408            )
6409            .await
6410            .unwrap();
6411
6412        let mut events = runtime.subscribe_events();
6413        runtime
6414            .start_turn(StartTurnRequest {
6415                thread_id: thread_id.clone(),
6416                message: "continue".to_string(),
6417                images: Vec::new(),
6418                provider_override: None,
6419                model_override: None,
6420                reasoning_override: None,
6421                workspace: test_workspace(),
6422                instructions: InstructionBundle::default(),
6423                developer_context: None,
6424                task_ledger_required: false,
6425            })
6426            .await
6427            .unwrap();
6428        loop {
6429            let envelope = tokio::time::timeout(std::time::Duration::from_secs(5), events.recv())
6430                .await
6431                .unwrap()
6432                .unwrap();
6433            if envelope.thread_id.as_deref() == Some(&thread_id)
6434                && matches!(envelope.event, RoderEvent::TurnCompleted(_))
6435            {
6436                break;
6437            }
6438        }
6439
6440        let request = captured.lock().unwrap().clone().unwrap();
6441        assert!(
6442            matches!(
6443                request.transcript.first(),
6444                Some(TranscriptItem::ContextCompaction(_))
6445            ),
6446            "provider request should start with a local emergency compaction item"
6447        );
6448        assert!(
6449            request.transcript.len() < 4,
6450            "provider request should not replay the full oversized prior transcript: {:?}",
6451            request.transcript
6452        );
6453
6454        let _ = std::fs::remove_dir_all(thread_root);
6455    }
6456
6457    #[tokio::test]
6458    async fn continue_after_context_window_failure_compacts_before_provider_request() {
6459        let captured = Arc::new(StdMutex::new(None));
6460        let mut builder = ExtensionRegistryBuilder::new();
6461        builder.inference_engine(Arc::new(CapturingEngine {
6462            request: captured.clone(),
6463        }));
6464        let thread_root = std::env::temp_dir().join(format!(
6465            "roder-context-failure-continue-{}",
6466            uuid::Uuid::new_v4()
6467        ));
6468        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
6469            base_path: thread_root.clone(),
6470        }));
6471        let runtime = Arc::new(
6472            Runtime::new(
6473                builder.build().unwrap(),
6474                RuntimeConfig {
6475                    default_provider: PROVIDER_MOCK.to_string(),
6476                    default_model: "gpt-5.5".to_string(),
6477                    file_backed_dynamic_context: true,
6478                    ..RuntimeConfig::default()
6479                },
6480            )
6481            .unwrap(),
6482        );
6483        let thread_id = runtime
6484            .create_thread(Some("Context failure continue".to_string()))
6485            .await
6486            .unwrap()
6487            .thread_id;
6488        let failed_turn = "failed-turn".to_string();
6489        runtime
6490            .persist_turn_item(
6491                &thread_id,
6492                &failed_turn,
6493                &TranscriptItem::UserMessage(UserMessage::text("old work ".repeat(10_000))),
6494            )
6495            .await
6496            .unwrap();
6497        runtime
6498            .persist_turn_item(
6499                &thread_id,
6500                &failed_turn,
6501                &TranscriptItem::Error(ErrorRecord {
6502                    message: "Your input exceeds the context window of this model. Please adjust your input and try again."
6503                        .to_string(),
6504                }),
6505            )
6506            .await
6507            .unwrap();
6508
6509        let mut events = runtime.subscribe_events();
6510        runtime
6511            .start_turn(StartTurnRequest {
6512                thread_id: thread_id.clone(),
6513                message: "continue".to_string(),
6514                images: Vec::new(),
6515                provider_override: None,
6516                model_override: None,
6517                reasoning_override: None,
6518                workspace: test_workspace(),
6519                instructions: InstructionBundle::default(),
6520                developer_context: None,
6521                task_ledger_required: false,
6522            })
6523            .await
6524            .unwrap();
6525        loop {
6526            let envelope = tokio::time::timeout(std::time::Duration::from_secs(5), events.recv())
6527                .await
6528                .unwrap()
6529                .unwrap();
6530            if envelope.thread_id.as_deref() == Some(&thread_id)
6531                && matches!(envelope.event, RoderEvent::TurnCompleted(_))
6532            {
6533                break;
6534            }
6535        }
6536
6537        let request = captured.lock().unwrap().clone().unwrap();
6538        assert!(
6539            matches!(
6540                request.transcript.first(),
6541                Some(TranscriptItem::ContextCompaction(_))
6542            ),
6543            "provider request after context-window failure should start with local compaction"
6544        );
6545        assert!(
6546            request
6547                .transcript
6548                .iter()
6549                .any(|item| matches!(item, TranscriptItem::UserMessage(message) if message.text == "continue")),
6550            "current continue prompt must be preserved: {:?}",
6551            request.transcript
6552        );
6553        assert!(
6554            !request.transcript.iter().any(
6555                |item| matches!(item, TranscriptItem::Error(error) if error.message.contains("context window"))
6556            ),
6557            "raw prior context-window error should be summarized, not replayed: {:?}",
6558            request.transcript
6559        );
6560
6561        let _ = std::fs::remove_dir_all(thread_root);
6562    }
6563
6564    #[tokio::test]
6565    async fn workspace_for_thread_falls_back_when_metadata_is_missing() {
6566        let workspace = test_workspace();
6567        let mut builder = ExtensionRegistryBuilder::new();
6568        builder.inference_engine(Arc::new(FakeInferenceEngine));
6569        builder.thread_store_factory(Arc::new(MetadataMissingStoreFactory));
6570        let runtime = Runtime::new(
6571            builder.build().unwrap(),
6572            RuntimeConfig {
6573                workspace: Some(workspace.clone()),
6574                ..RuntimeConfig::default()
6575            },
6576        )
6577        .unwrap();
6578
6579        let resolved = runtime
6580            .workspace_for_thread(&ThreadId::from("thread-workflow"))
6581            .await
6582            .unwrap();
6583
6584        assert_eq!(resolved, workspace);
6585    }
6586
6587    #[tokio::test]
6588    async fn automations_can_create_project_thread_with_model_overrides() {
6589        let runtime = Runtime::fake().unwrap();
6590        let workspace = std::env::temp_dir().join("project");
6591        let metadata = runtime
6592            .create_thread_with(CreateThreadRequest {
6593                title: Some("Automation: nightly status".to_string()),
6594                workspace: workspace.display().to_string(),
6595                workspace_id: None,
6596                root_id: None,
6597                provider: Some("mock".to_string()),
6598                model: Some("mock".to_string()),
6599                selection_mode: None,
6600                tool_allowlist: Vec::new(),
6601                developer_instructions: None,
6602                external_tools: Vec::new(),
6603                runner: None,
6604            })
6605            .await
6606            .unwrap();
6607
6608        assert_eq!(
6609            metadata.title.as_deref(),
6610            Some("Automation: nightly status")
6611        );
6612        assert_eq!(metadata.workspace, workspace.display().to_string());
6613        assert_eq!(metadata.provider.as_deref(), Some("mock"));
6614        assert_eq!(metadata.model.as_deref(), Some("mock"));
6615    }
6616
6617    #[tokio::test]
6618    async fn prior_provider_compaction_boundary_is_used_on_next_turn() {
6619        let captured = Arc::new(StdMutex::new(None));
6620        let mut builder = ExtensionRegistryBuilder::new();
6621        builder.inference_engine(Arc::new(CapturingEngine {
6622            request: captured.clone(),
6623        }));
6624        let thread_root = std::env::temp_dir().join(format!(
6625            "roder-provider-compaction-boundary-{}",
6626            uuid::Uuid::new_v4()
6627        ));
6628        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
6629            base_path: thread_root.clone(),
6630        }));
6631        let runtime = Arc::new(
6632            Runtime::new(
6633                builder.build().unwrap(),
6634                RuntimeConfig {
6635                    default_provider: PROVIDER_MOCK.to_string(),
6636                    default_model: "gpt-5.5".to_string(),
6637                    ..RuntimeConfig::default()
6638                },
6639            )
6640            .unwrap(),
6641        );
6642        let thread_id = runtime
6643            .create_thread(Some("Provider compaction boundary".to_string()))
6644            .await
6645            .unwrap()
6646            .thread_id;
6647        let old_turn = "old-turn".to_string();
6648        runtime
6649            .persist_turn_item(
6650                &thread_id,
6651                &old_turn,
6652                &TranscriptItem::UserMessage(UserMessage::text(
6653                    "old history that must not be replayed after provider compaction",
6654                )),
6655            )
6656            .await
6657            .unwrap();
6658        runtime
6659            .persist_turn_item(
6660                &thread_id,
6661                &old_turn,
6662                &TranscriptItem::AssistantMessage(AssistantMessage {
6663                    text: "old answer".to_string(),
6664                    phase: None,
6665                }),
6666            )
6667            .await
6668            .unwrap();
6669        runtime
6670            .persist_turn_item(
6671                &thread_id,
6672                &old_turn,
6673                &TranscriptItem::ProviderMetadata(serde_json::json!({
6674                    "output": [{
6675                        "id": "cmp_1",
6676                        "type": "compaction",
6677                        "encrypted_content": "opaque-state"
6678                    }]
6679                })),
6680            )
6681            .await
6682            .unwrap();
6683        runtime
6684            .persist_turn_item(
6685                &thread_id,
6686                &old_turn,
6687                &TranscriptItem::AssistantMessage(AssistantMessage {
6688                    text: "after compact".to_string(),
6689                    phase: None,
6690                }),
6691            )
6692            .await
6693            .unwrap();
6694
6695        let mut events = runtime.subscribe_events();
6696        runtime
6697            .start_turn(StartTurnRequest {
6698                thread_id: thread_id.clone(),
6699                message: "continue".to_string(),
6700                images: Vec::new(),
6701                provider_override: None,
6702                model_override: None,
6703                reasoning_override: None,
6704                workspace: test_workspace(),
6705                instructions: InstructionBundle::default(),
6706                developer_context: None,
6707                task_ledger_required: false,
6708            })
6709            .await
6710            .unwrap();
6711        loop {
6712            let envelope = tokio::time::timeout(std::time::Duration::from_secs(5), events.recv())
6713                .await
6714                .unwrap()
6715                .unwrap();
6716            if envelope.thread_id.as_deref() == Some(&thread_id)
6717                && matches!(envelope.event, RoderEvent::TurnCompleted(_))
6718            {
6719                break;
6720            }
6721        }
6722
6723        let request = captured.lock().unwrap().clone().unwrap();
6724        let compaction_idx = request.transcript.iter().position(|item| {
6725            matches!(
6726                item,
6727                TranscriptItem::ProviderMetadata(metadata)
6728                    if crate::compaction::provider_metadata_has_compaction(metadata)
6729            )
6730        });
6731        assert!(
6732            compaction_idx.is_some(),
6733            "next provider request should include the provider compaction item: {:?}",
6734            request.transcript
6735        );
6736        // Skill/context injectors may prepend non-history items, but no prior-turn
6737        // conversation may appear before the latest provider compaction boundary.
6738        let history_before_boundary =
6739            request
6740                .transcript
6741                .iter()
6742                .take(compaction_idx.unwrap())
6743                .any(|item| match item {
6744                    TranscriptItem::AssistantMessage(_)
6745                    | TranscriptItem::ToolCall(_)
6746                    | TranscriptItem::ToolResult(_) => true,
6747                    TranscriptItem::UserMessage(message) => {
6748                        !message.text.contains("<skills>") && message.text != "continue"
6749                    }
6750                    _ => false,
6751                });
6752        assert!(
6753            !history_before_boundary,
6754            "pre-provider-compaction conversation must not be replayed: {:?}",
6755            request.transcript
6756        );
6757        assert!(
6758            !request.transcript.iter().any(|item| {
6759                matches!(
6760                    item,
6761                    TranscriptItem::UserMessage(message)
6762                        if message.text.contains("old history that must not be replayed")
6763                )
6764            }),
6765            "pre-provider-compaction history must not be replayed: {:?}",
6766            request.transcript
6767        );
6768        assert!(
6769            request.transcript.iter().any(|item| {
6770                matches!(
6771                    item,
6772                    TranscriptItem::UserMessage(message) if message.text == "continue"
6773                )
6774            }),
6775            "current continue prompt must be preserved: {:?}",
6776            request.transcript
6777        );
6778        assert_eq!(
6779            request.runtime.auto_compact_token_limit,
6780            Some(945_000),
6781            "gpt-5.5 should keep server-side auto compaction configured"
6782        );
6783
6784        let _ = std::fs::remove_dir_all(thread_root);
6785    }
6786
6787    #[test]
6788    fn server_side_compaction_is_only_enabled_for_supported_models() {
6789        let cfg = RuntimeConfig {
6790            auto_compact_token_limit: Some(123_456),
6791            ..RuntimeConfig::default()
6792        };
6793
6794        assert_eq!(server_side_compaction_threshold(&cfg, "mock"), None);
6795        assert_eq!(
6796            server_side_compaction_threshold(&cfg, "codex-auto-review"),
6797            None
6798        );
6799    }
6800
6801    #[test]
6802    fn reasoning_is_disabled_for_models_without_reasoning_support() {
6803        let cfg = RuntimeConfig {
6804            reasoning: Some(REASONING_HIGH.to_string()),
6805            ..RuntimeConfig::default()
6806        };
6807
6808        assert_eq!(
6809            effective_reasoning_for_model(&cfg, "claude-haiku-4-5-20251001"),
6810            REASONING_NONE
6811        );
6812        assert_eq!(
6813            reasoning_for_model(&cfg, "claude-haiku-4-5-20251001"),
6814            ReasoningConfig::default()
6815        );
6816    }
6817
6818    #[test]
6819    fn unsupported_configured_reasoning_falls_back_to_model_default() {
6820        let cfg = RuntimeConfig {
6821            reasoning: Some(REASONING_MINIMAL.to_string()),
6822            ..RuntimeConfig::default()
6823        };
6824
6825        assert_eq!(
6826            effective_reasoning_for_model(&cfg, "gpt-5.5"),
6827            REASONING_MEDIUM
6828        );
6829    }
6830
6831    #[test]
6832    fn unsupported_configured_gemini_reasoning_is_rejected() {
6833        let mut builder = ExtensionRegistryBuilder::new();
6834        builder.inference_engine(std::sync::Arc::new(FakeInferenceEngine));
6835
6836        let err = match Runtime::new(
6837            builder.build().unwrap(),
6838            RuntimeConfig {
6839                default_model: "gemini-3.5-flash".to_string(),
6840                reasoning: Some(REASONING_XHIGH.to_string()),
6841                ..RuntimeConfig::default()
6842            },
6843        ) {
6844            Ok(_) => panic!("expected unsupported Gemini reasoning to be rejected"),
6845            Err(err) => err,
6846        };
6847
6848        assert!(
6849            err.to_string()
6850                .contains("model gemini-3.5-flash does not support reasoning effort xhigh")
6851        );
6852    }
6853
6854    #[tokio::test]
6855    async fn selecting_none_for_non_reasoning_model_preserves_stored_preference() {
6856        let runtime = Runtime::new(
6857            Runtime::fake().unwrap().registry,
6858            RuntimeConfig {
6859                reasoning: Some(REASONING_HIGH.to_string()),
6860                ..RuntimeConfig::default()
6861            },
6862        )
6863        .unwrap();
6864
6865        let cfg = runtime
6866            .select_provider(
6867                roder_api::catalog::PROVIDER_MOCK.to_string(),
6868                Some("claude-haiku-4-5-20251001".to_string()),
6869                Some(REASONING_NONE.to_string()),
6870            )
6871            .await
6872            .unwrap();
6873
6874        assert_eq!(cfg.reasoning.as_deref(), Some(REASONING_HIGH));
6875        assert_eq!(runtime.effective_reasoning().await, REASONING_NONE);
6876    }
6877
6878    #[tokio::test]
6879    async fn selecting_none_for_model_that_supports_none_updates_preference() {
6880        let runtime = Runtime::new(
6881            Runtime::fake().unwrap().registry,
6882            RuntimeConfig {
6883                reasoning: Some(REASONING_HIGH.to_string()),
6884                ..RuntimeConfig::default()
6885            },
6886        )
6887        .unwrap();
6888
6889        let cfg = runtime
6890            .select_provider(
6891                roder_api::catalog::PROVIDER_MOCK.to_string(),
6892                Some("mock".to_string()),
6893                Some(REASONING_NONE.to_string()),
6894            )
6895            .await
6896            .unwrap();
6897
6898        assert_eq!(cfg.reasoning.as_deref(), Some(REASONING_NONE));
6899    }
6900
6901    #[test]
6902    fn parallel_tool_calls_default_on_with_model_override() {
6903        assert!(parallel_tool_calls_for_model(
6904            &RuntimeConfig::default(),
6905            "custom-model"
6906        ));
6907
6908        let cfg = RuntimeConfig {
6909            model_parallel_tool_calls: std::collections::HashMap::from([(
6910                "custom-model".to_string(),
6911                false,
6912            )]),
6913            ..RuntimeConfig::default()
6914        };
6915
6916        assert!(!parallel_tool_calls_for_model(&cfg, "custom-model"));
6917        assert!(parallel_tool_calls_for_model(&cfg, "other-model"));
6918    }
6919
6920    #[test]
6921    fn profile_parallel_tool_calls_applies_between_config_and_default() {
6922        let cfg = RuntimeConfig {
6923            model_profiles: std::collections::HashMap::from([(
6924                "gpt-5.5".to_string(),
6925                test_model_profile("gpt-5.5"),
6926            )]),
6927            ..RuntimeConfig::default()
6928        };
6929
6930        assert!(!parallel_tool_calls_for_model(&cfg, "gpt-5.5"));
6931
6932        let cfg = RuntimeConfig {
6933            model_parallel_tool_calls: std::collections::HashMap::from([(
6934                "gpt-5.5".to_string(),
6935                true,
6936            )]),
6937            ..cfg
6938        };
6939
6940        assert!(parallel_tool_calls_for_model(&cfg, "gpt-5.5"));
6941    }
6942
6943    struct CapturingEngine {
6944        request: Arc<StdMutex<Option<AgentInferenceRequest>>>,
6945    }
6946
6947    #[async_trait::async_trait]
6948    impl InferenceEngine for CapturingEngine {
6949        fn id(&self) -> String {
6950            roder_api::catalog::PROVIDER_MOCK.to_string()
6951        }
6952
6953        fn capabilities(&self) -> InferenceCapabilities {
6954            InferenceCapabilities::coding_agent_default()
6955        }
6956
6957        async fn list_models(
6958            &self,
6959            _ctx: InferenceProviderContext<'_>,
6960        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
6961            Ok(roder_api::catalog::models_for_provider(
6962                roder_api::catalog::PROVIDER_MOCK,
6963                true,
6964            ))
6965        }
6966
6967        async fn stream_turn(
6968            &self,
6969            _ctx: InferenceTurnContext<'_>,
6970            request: AgentInferenceRequest,
6971        ) -> anyhow::Result<InferenceEventStream> {
6972            *self.request.lock().unwrap() = Some(request);
6973            Ok(Box::pin(stream::iter(vec![
6974                Ok(InferenceEvent::MessageDelta(MessageDelta {
6975                    text: "done".to_string(),
6976                    phase: None,
6977                })),
6978                Ok(InferenceEvent::Completed(CompletionMetadata {
6979                    stop_reason: Some("stop".to_string()),
6980                    provider_response_id: None,
6981                })),
6982            ])))
6983        }
6984    }
6985
6986    struct RoutingCaptureEngine {
6987        id: &'static str,
6988        models: Vec<ModelDescriptor>,
6989        requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
6990    }
6991
6992    #[async_trait::async_trait]
6993    impl InferenceEngine for RoutingCaptureEngine {
6994        fn id(&self) -> String {
6995            self.id.to_string()
6996        }
6997
6998        fn capabilities(&self) -> InferenceCapabilities {
6999            InferenceCapabilities::coding_agent_default()
7000        }
7001
7002        async fn list_models(
7003            &self,
7004            _ctx: InferenceProviderContext<'_>,
7005        ) -> anyhow::Result<Vec<ModelDescriptor>> {
7006            Ok(self.models.clone())
7007        }
7008
7009        async fn stream_turn(
7010            &self,
7011            _ctx: InferenceTurnContext<'_>,
7012            request: AgentInferenceRequest,
7013        ) -> anyhow::Result<InferenceEventStream> {
7014            self.requests.lock().unwrap().push(request);
7015            Ok(Box::pin(stream::iter(vec![
7016                Ok(InferenceEvent::MessageDelta(MessageDelta {
7017                    text: "routed".to_string(),
7018                    phase: None,
7019                })),
7020                Ok(InferenceEvent::Completed(CompletionMetadata {
7021                    stop_reason: Some("stop".to_string()),
7022                    provider_response_id: None,
7023                })),
7024            ])))
7025        }
7026    }
7027
7028    struct StaticRouter {
7029        id: &'static str,
7030        decision: InferenceRoutingDecision,
7031        contexts: Arc<StdMutex<Vec<InferenceRoutingContext>>>,
7032    }
7033
7034    #[async_trait::async_trait]
7035    impl InferenceRouter for StaticRouter {
7036        fn id(&self) -> String {
7037            self.id.to_string()
7038        }
7039
7040        async fn route(
7041            &self,
7042            context: InferenceRoutingContext,
7043        ) -> anyhow::Result<InferenceRoutingDecision> {
7044            self.contexts.lock().unwrap().push(context);
7045            Ok(self.decision.clone())
7046        }
7047    }
7048
7049    fn routing_test_model(id: &str, supported_reasoning: &[&str]) -> ModelDescriptor {
7050        ModelDescriptor {
7051            id: id.to_string(),
7052            name: id.to_string(),
7053            context_window: Some(128_000),
7054            default_reasoning: supported_reasoning
7055                .first()
7056                .map(|effort| (*effort).to_string()),
7057            supported_reasoning: supported_reasoning
7058                .iter()
7059                .map(|effort| ReasoningEffortDescriptor {
7060                    effort: (*effort).to_string(),
7061                    description: format!("{effort} reasoning"),
7062                })
7063                .collect(),
7064        }
7065    }
7066
7067    struct TaskLedgerCompletionGateEngine {
7068        calls: StdMutex<u32>,
7069        requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
7070    }
7071
7072    #[async_trait::async_trait]
7073    impl InferenceEngine for TaskLedgerCompletionGateEngine {
7074        fn id(&self) -> String {
7075            roder_api::catalog::PROVIDER_MOCK.to_string()
7076        }
7077
7078        fn capabilities(&self) -> InferenceCapabilities {
7079            InferenceCapabilities::coding_agent_default()
7080        }
7081
7082        async fn list_models(
7083            &self,
7084            _ctx: InferenceProviderContext<'_>,
7085        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7086            Ok(roder_api::catalog::models_for_provider(
7087                roder_api::catalog::PROVIDER_MOCK,
7088                true,
7089            ))
7090        }
7091
7092        async fn stream_turn(
7093            &self,
7094            _ctx: InferenceTurnContext<'_>,
7095            request: AgentInferenceRequest,
7096        ) -> anyhow::Result<InferenceEventStream> {
7097            self.requests.lock().unwrap().push(request);
7098            let mut calls = self.calls.lock().unwrap();
7099            *calls += 1;
7100            let events = match *calls {
7101                1 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7102                    id: "ledger-open".to_string(),
7103                    name: TASK_LEDGER_TOOL_NAME.to_string(),
7104                    arguments: serde_json::json!({
7105                        "tasks": [
7106                            {
7107                                "id": "inspect",
7108                                "content": "Inspect local assets",
7109                                "status": "completed",
7110                                "evidence": "listed workspace"
7111                            },
7112                            {
7113                                "id": "write",
7114                                "content": "Write /app/result.txt",
7115                                "status": "pending"
7116                            }
7117                        ],
7118                        "requireCompletionEvidence": true
7119                    })
7120                    .to_string(),
7121                }))],
7122                3 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7123                    id: "ledger-complete".to_string(),
7124                    name: TASK_LEDGER_TOOL_NAME.to_string(),
7125                    arguments: serde_json::json!({
7126                        "tasks": [
7127                            {
7128                                "id": "inspect",
7129                                "content": "Inspect local assets",
7130                                "status": "completed",
7131                                "evidence": "listed workspace"
7132                            },
7133                            {
7134                                "id": "write",
7135                                "content": "Write /app/result.txt",
7136                                "status": "completed",
7137                                "evidence": "wrote answer"
7138                            }
7139                        ],
7140                        "requireCompletionEvidence": true
7141                    })
7142                    .to_string(),
7143                }))],
7144                _ => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
7145                    text: "final".to_string(),
7146                    phase: None,
7147                }))],
7148            };
7149            Ok(Box::pin(stream::iter(events.into_iter().chain(
7150                std::iter::once(Ok(InferenceEvent::Completed(CompletionMetadata {
7151                    stop_reason: Some("stop".to_string()),
7152                    provider_response_id: None,
7153                }))),
7154            ))))
7155        }
7156    }
7157
7158    struct VerificationGateEngine {
7159        calls: StdMutex<u32>,
7160    }
7161
7162    #[async_trait::async_trait]
7163    impl InferenceEngine for VerificationGateEngine {
7164        fn id(&self) -> String {
7165            roder_api::catalog::PROVIDER_MOCK.to_string()
7166        }
7167
7168        fn capabilities(&self) -> InferenceCapabilities {
7169            InferenceCapabilities::coding_agent_default()
7170        }
7171
7172        async fn list_models(
7173            &self,
7174            _ctx: InferenceProviderContext<'_>,
7175        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7176            Ok(roder_api::catalog::models_for_provider(
7177                roder_api::catalog::PROVIDER_MOCK,
7178                true,
7179            ))
7180        }
7181
7182        async fn stream_turn(
7183            &self,
7184            _ctx: InferenceTurnContext<'_>,
7185            request: AgentInferenceRequest,
7186        ) -> anyhow::Result<InferenceEventStream> {
7187            let mut calls = self.calls.lock().unwrap();
7188            *calls += 1;
7189            let events = match *calls {
7190                1 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7191                    id: "write-1".to_string(),
7192                    name: "write_file".to_string(),
7193                    arguments: serde_json::json!({
7194                        "path": "src/lib.rs",
7195                        "content": "pub fn answer() -> u8 { 42 }\n"
7196                    })
7197                    .to_string(),
7198                }))],
7199                2 => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
7200                    text: "done too early".to_string(),
7201                    phase: None,
7202                }))],
7203                3 if request.transcript.iter().any(|item| {
7204                    matches!(
7205                        item,
7206                        TranscriptItem::UserMessage(message)
7207                            if message.text.contains("Verification gate blocked final completion")
7208                    )
7209                }) =>
7210                {
7211                    vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7212                        id: "verify-1".to_string(),
7213                        name: crate::verification_gate::VERIFICATION_TOOL_NAME.to_string(),
7214                        arguments: serde_json::json!({
7215                            "originalTask": "write code",
7216                            "changedFiles": ["src/lib.rs"],
7217                            "toolEvidence": ["write_file wrote src/lib.rs"],
7218                            "testsRun": ["cargo test -p roder-core verification_gate"],
7219                            "openGaps": [],
7220                            "status": "completed"
7221                        })
7222                        .to_string(),
7223                    }))]
7224                }
7225                _ => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
7226                    text: "verified final".to_string(),
7227                    phase: None,
7228                }))],
7229            };
7230            Ok(Box::pin(stream::iter(events.into_iter().chain(
7231                std::iter::once(Ok(InferenceEvent::Completed(CompletionMetadata {
7232                    stop_reason: Some("stop".to_string()),
7233                    provider_response_id: None,
7234                }))),
7235            ))))
7236        }
7237    }
7238
7239    struct SpeedPolicyEngine {
7240        calls: StdMutex<u32>,
7241        requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
7242    }
7243
7244    #[async_trait::async_trait]
7245    impl InferenceEngine for SpeedPolicyEngine {
7246        fn id(&self) -> String {
7247            roder_api::catalog::PROVIDER_MOCK.to_string()
7248        }
7249
7250        fn capabilities(&self) -> InferenceCapabilities {
7251            InferenceCapabilities::coding_agent_default()
7252        }
7253
7254        async fn list_models(
7255            &self,
7256            _ctx: InferenceProviderContext<'_>,
7257        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7258            Ok(roder_api::catalog::models_for_provider(
7259                roder_api::catalog::PROVIDER_MOCK,
7260                true,
7261            ))
7262        }
7263
7264        async fn stream_turn(
7265            &self,
7266            _ctx: InferenceTurnContext<'_>,
7267            request: AgentInferenceRequest,
7268        ) -> anyhow::Result<InferenceEventStream> {
7269            self.requests.lock().unwrap().push(request.clone());
7270            let mut calls = self.calls.lock().unwrap();
7271            *calls += 1;
7272            let events = match *calls {
7273                1 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7274                    id: "write-1".to_string(),
7275                    name: "write_file".to_string(),
7276                    arguments: serde_json::json!({
7277                        "path": "src/lib.rs",
7278                        "content": "pub fn answer() -> u8 { 42 }\n"
7279                    })
7280                    .to_string(),
7281                }))],
7282                3 if request.transcript.iter().any(|item| {
7283                    matches!(
7284                        item,
7285                        TranscriptItem::UserMessage(message)
7286                            if message.text.contains("Verification gate blocked final completion")
7287                    )
7288                }) =>
7289                {
7290                    vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7291                        id: "verify-1".to_string(),
7292                        name: crate::verification_gate::VERIFICATION_TOOL_NAME.to_string(),
7293                        arguments: serde_json::json!({
7294                            "originalTask": "write code",
7295                            "changedFiles": ["src/lib.rs"],
7296                            "toolEvidence": ["write_file wrote src/lib.rs"],
7297                            "testsRun": ["cargo test -p roder-core speed_policy"],
7298                            "openGaps": [],
7299                            "status": "completed"
7300                        })
7301                        .to_string(),
7302                    }))]
7303                }
7304                _ => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
7305                    text: "done".to_string(),
7306                    phase: None,
7307                }))],
7308            };
7309            Ok(Box::pin(stream::iter(events.into_iter().chain(
7310                std::iter::once(Ok(InferenceEvent::Completed(CompletionMetadata {
7311                    stop_reason: Some("stop".to_string()),
7312                    provider_response_id: None,
7313                }))),
7314            ))))
7315        }
7316    }
7317
7318    struct SwitchCaptureEngine {
7319        requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
7320    }
7321
7322    #[async_trait::async_trait]
7323    impl InferenceEngine for SwitchCaptureEngine {
7324        fn id(&self) -> String {
7325            roder_api::catalog::PROVIDER_MOCK.to_string()
7326        }
7327
7328        fn capabilities(&self) -> InferenceCapabilities {
7329            InferenceCapabilities::coding_agent_default()
7330        }
7331
7332        async fn list_models(
7333            &self,
7334            _ctx: InferenceProviderContext<'_>,
7335        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7336            Ok(roder_api::catalog::models_for_provider(
7337                roder_api::catalog::PROVIDER_MOCK,
7338                true,
7339            ))
7340        }
7341
7342        async fn stream_turn(
7343            &self,
7344            _ctx: InferenceTurnContext<'_>,
7345            request: AgentInferenceRequest,
7346        ) -> anyhow::Result<InferenceEventStream> {
7347            self.requests.lock().unwrap().push(request);
7348            Ok(Box::pin(stream::iter(vec![
7349                Ok(InferenceEvent::MessageDelta(MessageDelta {
7350                    text: "done".to_string(),
7351                    phase: None,
7352                })),
7353                Ok(InferenceEvent::Completed(CompletionMetadata {
7354                    stop_reason: Some("stop".to_string()),
7355                    provider_response_id: None,
7356                })),
7357            ])))
7358        }
7359    }
7360
7361    struct DeadlineEngine;
7362
7363    #[async_trait::async_trait]
7364    impl InferenceEngine for DeadlineEngine {
7365        fn id(&self) -> String {
7366            roder_api::catalog::PROVIDER_MOCK.to_string()
7367        }
7368
7369        fn capabilities(&self) -> InferenceCapabilities {
7370            InferenceCapabilities::coding_agent_default()
7371        }
7372
7373        async fn list_models(
7374            &self,
7375            _ctx: InferenceProviderContext<'_>,
7376        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7377            Ok(Vec::new())
7378        }
7379
7380        async fn stream_turn(
7381            &self,
7382            _ctx: InferenceTurnContext<'_>,
7383            _request: AgentInferenceRequest,
7384        ) -> anyhow::Result<InferenceEventStream> {
7385            Ok(Box::pin(stream::once(async {
7386                tokio::time::sleep(std::time::Duration::from_secs(60)).await;
7387                Ok(InferenceEvent::MessageDelta(MessageDelta {
7388                    text: "too late".to_string(),
7389                    phase: None,
7390                }))
7391            })))
7392        }
7393    }
7394
7395    struct WriteFileContributor;
7396
7397    impl ToolContributor for WriteFileContributor {
7398        fn id(&self) -> String {
7399            "test-write".to_string()
7400        }
7401
7402        fn contribute(&self, registry: &mut ToolRegistry) -> anyhow::Result<()> {
7403            registry.register(Arc::new(WriteFileTool))
7404        }
7405    }
7406
7407    struct WriteFileTool;
7408
7409    #[async_trait::async_trait]
7410    impl ToolExecutor for WriteFileTool {
7411        fn spec(&self) -> ToolSpec {
7412            ToolSpec {
7413                name: "write_file".to_string(),
7414                description: "Write a test file.".to_string(),
7415                parameters: serde_json::json!({
7416                    "type": "object",
7417                    "properties": {
7418                        "path": { "type": "string" },
7419                        "content": { "type": "string" }
7420                    },
7421                    "required": ["path", "content"],
7422                    "additionalProperties": false
7423                }),
7424            }
7425        }
7426
7427        async fn execute(
7428            &self,
7429            _ctx: ToolExecutionContext,
7430            call: ToolCall,
7431        ) -> anyhow::Result<ToolResult> {
7432            let path = call
7433                .arguments
7434                .get("path")
7435                .and_then(serde_json::Value::as_str)
7436                .unwrap_or("src/lib.rs");
7437            Ok(ToolResult {
7438                id: call.id,
7439                name: call.name,
7440                text: format!("wrote {path}"),
7441                data: serde_json::json!({ "path": path }),
7442                is_error: false,
7443            })
7444        }
7445    }
7446
7447    struct ProfileToolContributor;
7448
7449    impl ToolContributor for ProfileToolContributor {
7450        fn id(&self) -> String {
7451            "profile-tools".to_string()
7452        }
7453
7454        fn contribute(&self, registry: &mut ToolRegistry) -> anyhow::Result<()> {
7455            for name in ["apply_patch", "edit", "multi_edit", "write_file"] {
7456                registry.register(Arc::new(ProfileTool {
7457                    name: name.to_string(),
7458                }))?;
7459            }
7460            Ok(())
7461        }
7462    }
7463
7464    struct ProfileTool {
7465        name: String,
7466    }
7467
7468    #[async_trait::async_trait]
7469    impl ToolExecutor for ProfileTool {
7470        fn spec(&self) -> ToolSpec {
7471            ToolSpec {
7472                name: self.name.clone(),
7473                description: format!("{} test tool", self.name),
7474                parameters: serde_json::json!({
7475                    "type": "object",
7476                    "properties": {
7477                        "path": { "type": "string" },
7478                        "content": { "type": "string" }
7479                    },
7480                    "required": ["path", "content"],
7481                    "additionalProperties": false
7482                }),
7483            }
7484        }
7485
7486        async fn execute(
7487            &self,
7488            _ctx: ToolExecutionContext,
7489            call: ToolCall,
7490        ) -> anyhow::Result<ToolResult> {
7491            Ok(ToolResult {
7492                id: call.id,
7493                name: call.name,
7494                text: "ok".to_string(),
7495                data: serde_json::json!({}),
7496                is_error: false,
7497            })
7498        }
7499    }
7500
7501    fn test_model_profile(model: &str) -> ModelHarnessProfile {
7502        ModelHarnessProfile {
7503            model: model.to_string(),
7504            provider: roder_api::catalog::PROVIDER_OPENAI.to_string(),
7505            provider_family: ProviderFamily::OpenAi,
7506            edit_tool: Some(EDIT_TOOL_EDIT.to_string()),
7507            schema_policy: ModelSchemaPolicy::StandardRequiredFirst,
7508            instruction_overlay: ModelInstructionOverlay::IntuitiveContext,
7509            reasoning: ModelProfileReasoning {
7510                orientation: Some(REASONING_LOW.to_string()),
7511                execution: Some(REASONING_LOW.to_string()),
7512                verification: Some(REASONING_LOW.to_string()),
7513                recovery: Some(REASONING_LOW.to_string()),
7514            },
7515            parallel_tool_calls: Some(false),
7516            auto_compact_token_limit: Some(123_000),
7517        }
7518    }
7519
7520    async fn captured_profile_request(cfg: RuntimeConfig) -> AgentInferenceRequest {
7521        let captured = Arc::new(StdMutex::new(None));
7522        let mut builder = ExtensionRegistryBuilder::new();
7523        builder.inference_engine(Arc::new(CapturingEngine {
7524            request: captured.clone(),
7525        }));
7526        builder.tool_contributor(Arc::new(ProfileToolContributor));
7527        let runtime = Arc::new(Runtime::new(builder.build().unwrap(), cfg).unwrap());
7528        let mut rx = runtime.subscribe_events();
7529        let turn_id = runtime
7530            .start_turn(StartTurnRequest {
7531                thread_id: "thread-model-profile".to_string(),
7532                message: "use profile knobs".to_string(),
7533                images: Vec::new(),
7534                provider_override: None,
7535                model_override: None,
7536                reasoning_override: None,
7537                workspace: test_workspace(),
7538                instructions: InstructionBundle {
7539                    system: None,
7540                    developer: Some("base developer".to_string()),
7541                    developer_context: None,
7542                },
7543                developer_context: None,
7544                task_ledger_required: false,
7545            })
7546            .await
7547            .unwrap();
7548
7549        tokio::time::timeout(std::time::Duration::from_secs(5), async {
7550            loop {
7551                let envelope = rx.recv().await.unwrap();
7552                if envelope.turn_id.as_deref() != Some(&turn_id) {
7553                    continue;
7554                }
7555                match envelope.event {
7556                    RoderEvent::TurnCompleted(_) => break,
7557                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7558                    _ => {}
7559                }
7560            }
7561        })
7562        .await
7563        .unwrap();
7564
7565        captured.lock().unwrap().clone().unwrap()
7566    }
7567
7568    struct ToolThenStopEngine {
7569        calls: StdMutex<u32>,
7570    }
7571
7572    #[async_trait::async_trait]
7573    impl InferenceEngine for ToolThenStopEngine {
7574        fn id(&self) -> String {
7575            roder_api::catalog::PROVIDER_MOCK.to_string()
7576        }
7577
7578        fn capabilities(&self) -> InferenceCapabilities {
7579            InferenceCapabilities::coding_agent_default()
7580        }
7581
7582        async fn list_models(
7583            &self,
7584            _ctx: InferenceProviderContext<'_>,
7585        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7586            Ok(roder_api::catalog::models_for_provider(
7587                roder_api::catalog::PROVIDER_MOCK,
7588                true,
7589            ))
7590        }
7591
7592        async fn stream_turn(
7593            &self,
7594            _ctx: InferenceTurnContext<'_>,
7595            _request: AgentInferenceRequest,
7596        ) -> anyhow::Result<InferenceEventStream> {
7597            let mut calls = self.calls.lock().unwrap();
7598            *calls += 1;
7599            let events = match *calls {
7600                1 => vec![
7601                    Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7602                        id: "write-1".to_string(),
7603                        name: "write_file".to_string(),
7604                        arguments: serde_json::json!({
7605                            "path": "src/lib.rs",
7606                            "content": "pub fn answer() -> u8 { 42 }\n"
7607                        })
7608                        .to_string(),
7609                    })),
7610                    Ok(InferenceEvent::Completed(CompletionMetadata {
7611                        stop_reason: Some("tool_use".to_string()),
7612                        provider_response_id: None,
7613                    })),
7614                ],
7615                _ => vec![
7616                    Ok(InferenceEvent::MessageDelta(MessageDelta {
7617                        text: "final".to_string(),
7618                        phase: None,
7619                    })),
7620                    Ok(InferenceEvent::Completed(CompletionMetadata {
7621                        stop_reason: Some("end_turn".to_string()),
7622                        provider_response_id: None,
7623                    })),
7624                ],
7625            };
7626            Ok(Box::pin(stream::iter(events)))
7627        }
7628    }
7629
7630    #[tokio::test]
7631    async fn turn_completed_reports_terminal_step_finish_reason() {
7632        let mut builder = ExtensionRegistryBuilder::new();
7633        builder.inference_engine(Arc::new(ToolThenStopEngine {
7634            calls: StdMutex::new(0),
7635        }));
7636        builder.tool_contributor(Arc::new(WriteFileContributor));
7637        let runtime = Arc::new(
7638            Runtime::new(
7639                builder.build().unwrap(),
7640                RuntimeConfig {
7641                    policy_mode: PolicyMode::Bypass,
7642                    agent_swarm_mode: false,
7643                    ultra_mode: false,
7644                    ..RuntimeConfig::default()
7645                },
7646            )
7647            .unwrap(),
7648        );
7649        let mut rx = runtime.subscribe_events();
7650        let turn_id = runtime
7651            .start_turn(StartTurnRequest {
7652                thread_id: "thread-finish-reason".to_string(),
7653                message: "write then finish".to_string(),
7654                images: Vec::new(),
7655                provider_override: None,
7656                model_override: None,
7657                reasoning_override: None,
7658                workspace: test_workspace(),
7659                instructions: InstructionBundle {
7660                    system: None,
7661                    developer: None,
7662                    developer_context: None,
7663                },
7664                developer_context: None,
7665                task_ledger_required: false,
7666            })
7667            .await
7668            .unwrap();
7669
7670        let completed = tokio::time::timeout(std::time::Duration::from_secs(5), async {
7671            loop {
7672                let envelope = rx.recv().await.unwrap();
7673                if envelope.turn_id.as_deref() != Some(&turn_id) {
7674                    continue;
7675                }
7676                match envelope.event {
7677                    RoderEvent::TurnCompleted(event) => break event,
7678                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7679                    _ => {}
7680                }
7681            }
7682        })
7683        .await
7684        .unwrap();
7685
7686        // The mid-turn tool_use stop reason must not leak; the terminal
7687        // end_turn step decides the turn's finish reason.
7688        assert_eq!(completed.finish_reason.as_deref(), Some("stop"));
7689    }
7690
7691    #[tokio::test]
7692    async fn inference_router_selection_changes_request_model_and_records_event() {
7693        let default_requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7694        let routed_requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7695        let contexts = Arc::new(StdMutex::new(Vec::<InferenceRoutingContext>::new()));
7696        let selected = ModelSelection {
7697            provider: "routed-provider".to_string(),
7698            model: "routed-model".to_string(),
7699        };
7700        let default = ModelSelection {
7701            provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7702            model: "mock".to_string(),
7703        };
7704        let decision = InferenceRoutingDecision {
7705            reasoning: Some(ReasoningConfig {
7706                enabled: true,
7707                level: Some(REASONING_LOW.to_string()),
7708            }),
7709            confidence: Some(0.91),
7710            baseline: Some(default.clone()),
7711            matched_signals: vec![roder_api::inference_routing::InferenceRoutingSignal::new(
7712                "intent", "routine",
7713            )],
7714            ..InferenceRoutingDecision::selected("test-router", selected.clone(), "routine request")
7715        };
7716
7717        let mut builder = ExtensionRegistryBuilder::new();
7718        builder.inference_engine(Arc::new(RoutingCaptureEngine {
7719            id: roder_api::catalog::PROVIDER_MOCK,
7720            models: vec![routing_test_model("mock", &[REASONING_LOW])],
7721            requests: default_requests.clone(),
7722        }));
7723        builder.inference_engine(Arc::new(RoutingCaptureEngine {
7724            id: "routed-provider",
7725            models: vec![routing_test_model(
7726                "routed-model",
7727                &[REASONING_LOW, REASONING_MEDIUM],
7728            )],
7729            requests: routed_requests.clone(),
7730        }));
7731        builder.inference_router(Arc::new(StaticRouter {
7732            id: "test-router",
7733            decision,
7734            contexts: contexts.clone(),
7735        }));
7736        let thread_root =
7737            std::env::temp_dir().join(format!("roder-routing-auto-{}", uuid::Uuid::new_v4()));
7738        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
7739            base_path: thread_root.clone(),
7740        }));
7741        let runtime = Arc::new(
7742            Runtime::new(
7743                builder.build().unwrap(),
7744                RuntimeConfig {
7745                    default_provider: default.provider.clone(),
7746                    default_model: default.model.clone(),
7747                    ..RuntimeConfig::default()
7748                },
7749            )
7750            .unwrap(),
7751        );
7752        let thread_id = runtime
7753            .create_thread_with(CreateThreadRequest {
7754                title: Some("Routing auto".to_string()),
7755                workspace: test_workspace(),
7756                workspace_id: None,
7757                root_id: None,
7758                provider: Some(default.provider.clone()),
7759                model: Some(default.model.clone()),
7760                tool_allowlist: Vec::new(),
7761                developer_instructions: None,
7762                external_tools: Vec::new(),
7763                selection_mode: Some(ModelSelectionMode::auto(
7764                    "test-router:coding",
7765                    "test-router",
7766                    "Auto: Coding",
7767                    default.clone(),
7768                    Some("coding".to_string()),
7769                    None,
7770                )),
7771                runner: None,
7772            })
7773            .await
7774            .unwrap()
7775            .thread_id;
7776        let mut rx = runtime.subscribe_events();
7777        let turn_id = runtime
7778            .start_turn(StartTurnRequest {
7779                thread_id: thread_id.clone(),
7780                message: "small cleanup".to_string(),
7781                images: Vec::new(),
7782                provider_override: None,
7783                model_override: None,
7784                reasoning_override: None,
7785                workspace: test_workspace(),
7786                instructions: InstructionBundle::default(),
7787                developer_context: None,
7788                task_ledger_required: false,
7789            })
7790            .await
7791            .unwrap();
7792
7793        let mut routing_event = None;
7794        let mut inference_started = None;
7795        tokio::time::timeout(std::time::Duration::from_secs(5), async {
7796            loop {
7797                let envelope = rx.recv().await.unwrap();
7798                if envelope.turn_id.as_deref() != Some(&turn_id) {
7799                    continue;
7800                }
7801                match envelope.event {
7802                    RoderEvent::InferenceRoutingDecision(event) => {
7803                        routing_event = Some(event);
7804                    }
7805                    RoderEvent::InferenceStarted(event) => {
7806                        inference_started = Some(event);
7807                    }
7808                    RoderEvent::TurnCompleted(_) => break,
7809                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7810                    _ => {}
7811                }
7812            }
7813        })
7814        .await
7815        .unwrap();
7816
7817        assert!(default_requests.lock().unwrap().is_empty());
7818        let routed_requests = routed_requests.lock().unwrap();
7819        assert_eq!(routed_requests.len(), 1);
7820        assert_eq!(routed_requests[0].model, selected);
7821        assert_eq!(
7822            routed_requests[0].reasoning.level.as_deref(),
7823            Some(REASONING_LOW)
7824        );
7825        assert_eq!(
7826            routed_requests[0].metadata["inferenceRouting"]["outcome"],
7827            "selected"
7828        );
7829
7830        let routing_event = routing_event.expect("routing decision event");
7831        assert_eq!(routing_event.default_selection, default);
7832        assert_eq!(routing_event.selected_selection, selected);
7833        assert_eq!(
7834            routing_event.decision.outcome,
7835            InferenceRoutingOutcome::Selected
7836        );
7837        assert_eq!(
7838            inference_started.expect("inference started event").model,
7839            selected
7840        );
7841
7842        let contexts = contexts.lock().unwrap();
7843        assert_eq!(contexts.len(), 1);
7844        assert_eq!(contexts[0].default_selection, default);
7845        assert_eq!(contexts[0].candidates.len(), 2);
7846        assert!(
7847            contexts[0]
7848                .signals
7849                .iter()
7850                .any(|signal| signal.key == "profile" && signal.value == "coding")
7851        );
7852        let _ = std::fs::remove_dir_all(thread_root);
7853    }
7854
7855    #[tokio::test]
7856    async fn inference_router_is_bypassed_for_explicit_selection() {
7857        let requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7858        let contexts = Arc::new(StdMutex::new(Vec::<InferenceRoutingContext>::new()));
7859        let selected = ModelSelection {
7860            provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7861            model: "mock".to_string(),
7862        };
7863
7864        let mut builder = ExtensionRegistryBuilder::new();
7865        builder.inference_engine(Arc::new(RoutingCaptureEngine {
7866            id: roder_api::catalog::PROVIDER_MOCK,
7867            models: vec![routing_test_model("mock", &[REASONING_LOW])],
7868            requests: requests.clone(),
7869        }));
7870        builder.inference_router(Arc::new(StaticRouter {
7871            id: "test-router",
7872            decision: InferenceRoutingDecision::selected(
7873                "test-router",
7874                ModelSelection {
7875                    provider: "missing".to_string(),
7876                    model: "missing".to_string(),
7877                },
7878                "would route if called",
7879            ),
7880            contexts: contexts.clone(),
7881        }));
7882        let thread_root =
7883            std::env::temp_dir().join(format!("roder-routing-explicit-{}", uuid::Uuid::new_v4()));
7884        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
7885            base_path: thread_root.clone(),
7886        }));
7887        let runtime = Arc::new(
7888            Runtime::new(
7889                builder.build().unwrap(),
7890                RuntimeConfig {
7891                    default_provider: selected.provider.clone(),
7892                    default_model: selected.model.clone(),
7893                    inference_router: RuntimeInferenceRouterConfig {
7894                        enabled: true,
7895                        router_id: Some("test-router".to_string()),
7896                    },
7897                    ..RuntimeConfig::default()
7898                },
7899            )
7900            .unwrap(),
7901        );
7902        let thread_id = runtime
7903            .create_thread_with(CreateThreadRequest {
7904                title: Some("Routing explicit".to_string()),
7905                workspace: test_workspace(),
7906                workspace_id: None,
7907                root_id: None,
7908                provider: Some(selected.provider.clone()),
7909                model: Some(selected.model.clone()),
7910                tool_allowlist: Vec::new(),
7911                developer_instructions: None,
7912                external_tools: Vec::new(),
7913                selection_mode: Some(ModelSelectionMode::auto(
7914                    "test-router:default",
7915                    "test-router",
7916                    "Auto",
7917                    selected.clone(),
7918                    None,
7919                    None,
7920                )),
7921                runner: None,
7922            })
7923            .await
7924            .unwrap()
7925            .thread_id;
7926        let mut rx = runtime.subscribe_events();
7927        let turn_id = runtime
7928            .start_turn(StartTurnRequest {
7929                thread_id,
7930                message: "use explicit selection".to_string(),
7931                images: Vec::new(),
7932                provider_override: Some(selected.provider.clone()),
7933                model_override: Some(selected.model.clone()),
7934                reasoning_override: None,
7935                workspace: test_workspace(),
7936                instructions: InstructionBundle::default(),
7937                developer_context: None,
7938                task_ledger_required: false,
7939            })
7940            .await
7941            .unwrap();
7942
7943        let mut saw_routing_event = false;
7944        tokio::time::timeout(std::time::Duration::from_secs(5), async {
7945            loop {
7946                let envelope = rx.recv().await.unwrap();
7947                if envelope.turn_id.as_deref() != Some(&turn_id) {
7948                    continue;
7949                }
7950                match envelope.event {
7951                    RoderEvent::InferenceRoutingDecision(_) => {
7952                        saw_routing_event = true;
7953                    }
7954                    RoderEvent::TurnCompleted(_) => break,
7955                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7956                    _ => {}
7957                }
7958            }
7959        })
7960        .await
7961        .unwrap();
7962
7963        assert!(!saw_routing_event);
7964        assert!(contexts.lock().unwrap().is_empty());
7965        let requests = requests.lock().unwrap();
7966        assert_eq!(requests.len(), 1);
7967        assert_eq!(requests[0].model, selected);
7968        let _ = std::fs::remove_dir_all(thread_root);
7969    }
7970
7971    #[tokio::test]
7972    async fn inference_router_is_bypassed_for_manual_selection_mode() {
7973        let requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7974        let contexts = Arc::new(StdMutex::new(Vec::<InferenceRoutingContext>::new()));
7975        let selected = ModelSelection {
7976            provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7977            model: "mock".to_string(),
7978        };
7979
7980        let mut builder = ExtensionRegistryBuilder::new();
7981        builder.inference_engine(Arc::new(RoutingCaptureEngine {
7982            id: roder_api::catalog::PROVIDER_MOCK,
7983            models: vec![routing_test_model("mock", &[REASONING_LOW])],
7984            requests: requests.clone(),
7985        }));
7986        builder.inference_router(Arc::new(StaticRouter {
7987            id: "test-router",
7988            decision: InferenceRoutingDecision::selected(
7989                "test-router",
7990                ModelSelection {
7991                    provider: "missing".to_string(),
7992                    model: "missing".to_string(),
7993                },
7994                "would route if called",
7995            ),
7996            contexts: contexts.clone(),
7997        }));
7998        let thread_root =
7999            std::env::temp_dir().join(format!("roder-routing-manual-{}", uuid::Uuid::new_v4()));
8000        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8001            base_path: thread_root.clone(),
8002        }));
8003        let runtime = Arc::new(
8004            Runtime::new(
8005                builder.build().unwrap(),
8006                RuntimeConfig {
8007                    default_provider: selected.provider.clone(),
8008                    default_model: selected.model.clone(),
8009                    inference_router: RuntimeInferenceRouterConfig {
8010                        enabled: true,
8011                        router_id: Some("test-router".to_string()),
8012                    },
8013                    ..RuntimeConfig::default()
8014                },
8015            )
8016            .unwrap(),
8017        );
8018        let thread_id = runtime
8019            .create_thread_with(CreateThreadRequest {
8020                title: Some("Routing manual".to_string()),
8021                workspace: test_workspace(),
8022                workspace_id: None,
8023                root_id: None,
8024                provider: Some(selected.provider.clone()),
8025                model: Some(selected.model.clone()),
8026                tool_allowlist: Vec::new(),
8027                developer_instructions: None,
8028                external_tools: Vec::new(),
8029                selection_mode: Some(ModelSelectionMode::manual(
8030                    selected.provider.clone(),
8031                    selected.model.clone(),
8032                    None,
8033                )),
8034                runner: None,
8035            })
8036            .await
8037            .unwrap()
8038            .thread_id;
8039        let mut rx = runtime.subscribe_events();
8040        let turn_id = runtime
8041            .start_turn(StartTurnRequest {
8042                thread_id,
8043                message: "use selected manual model".to_string(),
8044                images: Vec::new(),
8045                provider_override: None,
8046                model_override: None,
8047                reasoning_override: None,
8048                workspace: test_workspace(),
8049                instructions: InstructionBundle::default(),
8050                developer_context: None,
8051                task_ledger_required: false,
8052            })
8053            .await
8054            .unwrap();
8055
8056        let mut saw_routing_event = false;
8057        tokio::time::timeout(std::time::Duration::from_secs(5), async {
8058            loop {
8059                let envelope = rx.recv().await.unwrap();
8060                if envelope.turn_id.as_deref() != Some(&turn_id) {
8061                    continue;
8062                }
8063                match envelope.event {
8064                    RoderEvent::InferenceRoutingDecision(_) => {
8065                        saw_routing_event = true;
8066                    }
8067                    RoderEvent::TurnCompleted(_) => break,
8068                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8069                    _ => {}
8070                }
8071            }
8072        })
8073        .await
8074        .unwrap();
8075
8076        assert!(!saw_routing_event);
8077        assert!(contexts.lock().unwrap().is_empty());
8078        let requests = requests.lock().unwrap();
8079        assert_eq!(requests.len(), 1);
8080        assert_eq!(requests[0].model, selected);
8081        let _ = std::fs::remove_dir_all(thread_root);
8082    }
8083
8084    #[test]
8085    fn enabled_inference_router_requires_registered_router() {
8086        let mut builder = ExtensionRegistryBuilder::new();
8087        builder.inference_engine(Arc::new(FakeInferenceEngine));
8088
8089        let err = match Runtime::new(
8090            builder.build().unwrap(),
8091            RuntimeConfig {
8092                inference_router: RuntimeInferenceRouterConfig {
8093                    enabled: true,
8094                    router_id: Some("missing-router".to_string()),
8095                },
8096                ..RuntimeConfig::default()
8097            },
8098        ) {
8099            Ok(_) => panic!("runtime should reject unknown inference router"),
8100            Err(err) => err,
8101        };
8102
8103        assert!(
8104            err.to_string()
8105                .contains("inference router \"missing-router\" is not registered")
8106        );
8107    }
8108
8109    #[tokio::test]
8110    async fn model_profile_routes_request_knobs_to_next_inference() {
8111        let request = captured_profile_request(RuntimeConfig {
8112            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8113            default_model: "gpt-5.5".to_string(),
8114            model_profiles: std::collections::HashMap::from([(
8115                "gpt-5.5".to_string(),
8116                test_model_profile("gpt-5.5"),
8117            )]),
8118            ..RuntimeConfig::default()
8119        })
8120        .await;
8121
8122        let tool_names = request
8123            .tools
8124            .iter()
8125            .map(|tool| tool.name.as_str())
8126            .collect::<Vec<_>>();
8127        assert!(tool_names.contains(&"apply_patch"));
8128        assert!(tool_names.contains(&"edit"));
8129        assert!(tool_names.contains(&"multi_edit"));
8130        assert!(tool_names.contains(&"write_file"));
8131        assert_eq!(request.reasoning.level.as_deref(), Some(REASONING_LOW));
8132        assert_eq!(request.runtime.parallel_tool_calls, Some(false));
8133        assert_eq!(request.runtime.auto_compact_token_limit, Some(123_000));
8134        assert!(
8135            request
8136                .instructions
8137                .developer
8138                .as_deref()
8139                .unwrap_or_default()
8140                .contains("Use the provided context as the current working set")
8141        );
8142        assert_eq!(
8143            request
8144                .metadata
8145                .pointer("/modelProfile/schemaPolicy")
8146                .and_then(serde_json::Value::as_str),
8147            Some("standard_required_first")
8148        );
8149    }
8150
8151    #[tokio::test]
8152    async fn ultra_reasoning_reaches_transport_state_and_enables_proactive_delegation() {
8153        let request = captured_profile_request(RuntimeConfig {
8154            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8155            default_model: "gpt-5.6-sol".to_string(),
8156            reasoning: Some(REASONING_ULTRA.to_string()),
8157            ..RuntimeConfig::default()
8158        })
8159        .await;
8160
8161        assert_eq!(request.reasoning.level.as_deref(), Some(REASONING_ULTRA));
8162        let developer = request
8163            .instructions
8164            .developer
8165            .as_deref()
8166            .expect("ultra developer instructions");
8167        assert!(developer.contains("Proactive multi-agent delegation is active"));
8168    }
8169
8170    #[tokio::test]
8171    async fn lower_sol_effort_requires_explicit_multi_agent_request() {
8172        let request = captured_profile_request(RuntimeConfig {
8173            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8174            default_model: "gpt-5.6-sol".to_string(),
8175            reasoning: Some(roder_api::catalog::REASONING_MEDIUM.to_string()),
8176            ..RuntimeConfig::default()
8177        })
8178        .await;
8179
8180        let developer = request
8181            .instructions
8182            .developer
8183            .as_deref()
8184            .expect("sol developer instructions");
8185        assert!(developer.contains("Do not spawn sub-agents unless"));
8186        assert!(!developer.contains("Proactive multi-agent delegation is active"));
8187    }
8188
8189    #[tokio::test]
8190    async fn luna_does_not_receive_codex_v2_multi_agent_policy() {
8191        let request = captured_profile_request(RuntimeConfig {
8192            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8193            default_model: "gpt-5.6-luna".to_string(),
8194            reasoning: Some(roder_api::catalog::REASONING_MAX.to_string()),
8195            ..RuntimeConfig::default()
8196        })
8197        .await;
8198
8199        let developer = request
8200            .instructions
8201            .developer
8202            .as_deref()
8203            .unwrap_or_default();
8204        assert!(!developer.contains("Do not spawn sub-agents unless"));
8205        assert!(!developer.contains("Proactive multi-agent delegation is active"));
8206    }
8207
8208    #[tokio::test]
8209    async fn ultra_mode_enables_proactive_multi_agent_for_any_model() {
8210        // Luna has no Ultra effort, but ultra mode still injects proactive policy.
8211        let request = captured_profile_request(RuntimeConfig {
8212            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8213            default_model: "gpt-5.6-luna".to_string(),
8214            reasoning: Some(roder_api::catalog::REASONING_MAX.to_string()),
8215            ultra_mode: true,
8216            ..RuntimeConfig::default()
8217        })
8218        .await;
8219
8220        let developer = request
8221            .instructions
8222            .developer
8223            .as_deref()
8224            .expect("ultra mode developer instructions");
8225        assert!(developer.contains("Proactive multi-agent delegation is active"));
8226        assert!(developer.contains("spawn_agent"));
8227    }
8228
8229    #[tokio::test]
8230    async fn ultra_mode_overrides_explicit_request_only_on_sol() {
8231        // Medium Sol effort is normally explicit-request-only; ultra mode flips
8232        // it to proactive without requiring Ultra reasoning effort.
8233        let request = captured_profile_request(RuntimeConfig {
8234            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8235            default_model: "gpt-5.6-sol".to_string(),
8236            reasoning: Some(roder_api::catalog::REASONING_MEDIUM.to_string()),
8237            ultra_mode: true,
8238            ..RuntimeConfig::default()
8239        })
8240        .await;
8241
8242        let developer = request
8243            .instructions
8244            .developer
8245            .as_deref()
8246            .expect("ultra mode sol developer instructions");
8247        assert!(developer.contains("Proactive multi-agent delegation is active"));
8248        assert!(!developer.contains("Do not spawn sub-agents unless"));
8249    }
8250
8251    #[tokio::test]
8252    async fn turn_developer_context_reaches_inference_and_does_not_persist() {
8253        let captured = Arc::new(StdMutex::new(None));
8254        let mut builder = ExtensionRegistryBuilder::new();
8255        builder.inference_engine(Arc::new(CapturingEngine {
8256            request: captured.clone(),
8257        }));
8258        let runtime = Arc::new(
8259            Runtime::new(
8260                builder.build().unwrap(),
8261                RuntimeConfig {
8262                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8263                    default_model: "gpt-5.5".to_string(),
8264                    ..RuntimeConfig::default()
8265                },
8266            )
8267            .unwrap(),
8268        );
8269
8270        async fn run_turn(runtime: &Arc<Runtime>, developer_context: Option<String>) {
8271            let mut rx = runtime.subscribe_events();
8272            let turn_id = runtime
8273                .start_turn(StartTurnRequest {
8274                    thread_id: "thread-turn-context".to_string(),
8275                    message: "hello".to_string(),
8276                    images: Vec::new(),
8277                    provider_override: None,
8278                    model_override: None,
8279                    reasoning_override: None,
8280                    workspace: test_workspace(),
8281                    instructions: InstructionBundle::default(),
8282                    developer_context,
8283                    task_ledger_required: false,
8284                })
8285                .await
8286                .unwrap();
8287            tokio::time::timeout(std::time::Duration::from_secs(5), async {
8288                loop {
8289                    let envelope = rx.recv().await.unwrap();
8290                    if envelope.turn_id.as_deref() != Some(&turn_id) {
8291                        continue;
8292                    }
8293                    match envelope.event {
8294                        RoderEvent::TurnCompleted(_) => break,
8295                        RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8296                        _ => {}
8297                    }
8298                }
8299            })
8300            .await
8301            .unwrap();
8302        }
8303
8304        run_turn(
8305            &runtime,
8306            Some("Connected accounts: example-service.".to_string()),
8307        )
8308        .await;
8309        let request = captured.lock().unwrap().clone().unwrap();
8310        assert_eq!(
8311            request.instructions.developer_context.as_deref(),
8312            Some("Connected accounts: example-service.")
8313        );
8314
8315        // The context is per-turn only: the next turn on the same thread
8316        // without a developerContext must not see the previous one.
8317        run_turn(&runtime, None).await;
8318        let request = captured.lock().unwrap().clone().unwrap();
8319        assert_eq!(request.instructions.developer_context, None);
8320    }
8321
8322    #[tokio::test]
8323    async fn tool_search_overrides_route_to_next_inference_request() {
8324        let request = captured_profile_request(RuntimeConfig {
8325            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8326            default_model: "gpt-5.4".to_string(),
8327            tool_search: ToolSearchConfig {
8328                mode: roder_api::inference::ToolSearchMode::Auto,
8329                max_catalog_items: Some(100),
8330                ..ToolSearchConfig::default()
8331            },
8332            provider_tool_search: std::collections::HashMap::from([(
8333                roder_api::catalog::PROVIDER_MOCK.to_string(),
8334                roder_api::inference::ToolSearchConfigOverlay {
8335                    include_skills: Some(false),
8336                    provider_variant: Some(roder_api::inference::ToolSearchProviderVariant::Regex),
8337                    ..Default::default()
8338                },
8339            )]),
8340            model_tool_search: std::collections::HashMap::from([(
8341                "gpt-5.4".to_string(),
8342                roder_api::inference::ToolSearchConfigOverlay {
8343                    mode: Some(roder_api::inference::ToolSearchMode::ProviderNative),
8344                    max_catalog_items: Some(25),
8345                    provider_variant: Some(roder_api::inference::ToolSearchProviderVariant::Bm25),
8346                    ..Default::default()
8347                },
8348            )]),
8349            ..RuntimeConfig::default()
8350        })
8351        .await;
8352
8353        assert_eq!(
8354            request.runtime.tool_search.mode,
8355            roder_api::inference::ToolSearchMode::ProviderNative
8356        );
8357        assert_eq!(request.runtime.tool_search.max_catalog_items, Some(25));
8358        assert_eq!(
8359            request.runtime.tool_search.provider_variant,
8360            roder_api::inference::ToolSearchProviderVariant::Bm25
8361        );
8362    }
8363
8364    #[tokio::test]
8365    async fn context_entrypoint_hints_use_turn_workspace() {
8366        let process_workspace = runtime_test_workspace("entrypoint-process");
8367        let thread_workspace = runtime_test_workspace("entrypoint-thread");
8368        std::fs::create_dir_all(process_workspace.join("src")).unwrap();
8369        std::fs::create_dir_all(thread_workspace.join("src")).unwrap();
8370        std::fs::write(
8371            process_workspace.join("src/sidebar-thread-groups.ts"),
8372            "export const desktopLeak = true;\n",
8373        )
8374        .unwrap();
8375        std::fs::write(
8376            thread_workspace.join("src/voice-plan-feedback.ts"),
8377            "export const voicePlanFeedback = true;\n",
8378        )
8379        .unwrap();
8380
8381        let captured = Arc::new(StdMutex::new(None));
8382        let mut builder = ExtensionRegistryBuilder::new();
8383        builder.inference_engine(Arc::new(CapturingEngine {
8384            request: captured.clone(),
8385        }));
8386        builder.context_planner(Arc::new(roder_context::EntrypointContextPlanner::new(
8387            process_workspace.clone(),
8388        )));
8389        let runtime = Arc::new(
8390            Runtime::new(
8391                builder.build().unwrap(),
8392                RuntimeConfig {
8393                    workspace: Some(process_workspace.display().to_string()),
8394                    ..RuntimeConfig::default()
8395                },
8396            )
8397            .unwrap(),
8398        );
8399        let mut rx = runtime.subscribe_events();
8400
8401        let turn_id = runtime
8402            .start_turn(StartTurnRequest {
8403                thread_id: "thread-workspace-entrypoint".to_string(),
8404                message: "investigate voice plan feedback".to_string(),
8405                images: Vec::new(),
8406                provider_override: None,
8407                model_override: None,
8408                reasoning_override: None,
8409                workspace: thread_workspace.display().to_string(),
8410                instructions: crate::instructions::default_instructions(),
8411                developer_context: None,
8412                task_ledger_required: false,
8413            })
8414            .await
8415            .unwrap();
8416
8417        tokio::time::timeout(std::time::Duration::from_secs(5), async {
8418            loop {
8419                let envelope = rx.recv().await.unwrap();
8420                if envelope.turn_id.as_deref() != Some(&turn_id) {
8421                    continue;
8422                }
8423                match envelope.event {
8424                    RoderEvent::TurnCompleted(_) => break,
8425                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8426                    _ => {}
8427                }
8428            }
8429        })
8430        .await
8431        .unwrap();
8432
8433        let request = captured.lock().unwrap().clone().expect("captured request");
8434        let transcript_text = request
8435            .transcript
8436            .iter()
8437            .map(|item| match item {
8438                TranscriptItem::UserMessage(message) => message.text.as_str(),
8439                _ => "",
8440            })
8441            .collect::<Vec<_>>()
8442            .join("\n");
8443        assert!(transcript_text.contains("src/voice-plan-feedback.ts"));
8444        assert!(!transcript_text.contains("src/sidebar-thread-groups.ts"));
8445
8446        let _ = std::fs::remove_dir_all(process_workspace);
8447        let _ = std::fs::remove_dir_all(thread_workspace);
8448    }
8449
8450    fn runtime_test_workspace(name: &str) -> std::path::PathBuf {
8451        let path =
8452            std::env::temp_dir().join(format!("roder-runtime-{name}-{}", uuid::Uuid::new_v4()));
8453        let _ = std::fs::remove_dir_all(&path);
8454        std::fs::create_dir_all(&path).unwrap();
8455        path
8456    }
8457
8458    #[tokio::test]
8459    async fn model_profile_user_model_knobs_override_profile_defaults() {
8460        let request = captured_profile_request(RuntimeConfig {
8461            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8462            default_model: "gpt-5.5".to_string(),
8463            reasoning: Some(REASONING_HIGH.to_string()),
8464            auto_compact_token_limit: Some(999),
8465            model_edit_tools: std::collections::HashMap::from([(
8466                "gpt-5.5".to_string(),
8467                EDIT_TOOL_PATCH.to_string(),
8468            )]),
8469            model_parallel_tool_calls: std::collections::HashMap::from([(
8470                "gpt-5.5".to_string(),
8471                true,
8472            )]),
8473            model_profiles: std::collections::HashMap::from([(
8474                "gpt-5.5".to_string(),
8475                test_model_profile("gpt-5.5"),
8476            )]),
8477            ..RuntimeConfig::default()
8478        })
8479        .await;
8480
8481        let tool_names = request
8482            .tools
8483            .iter()
8484            .map(|tool| tool.name.as_str())
8485            .collect::<Vec<_>>();
8486        assert!(tool_names.contains(&"apply_patch"));
8487        assert!(!tool_names.contains(&"edit"));
8488        assert_eq!(request.reasoning.level.as_deref(), Some(REASONING_HIGH));
8489        assert_eq!(request.runtime.parallel_tool_calls, Some(true));
8490        assert_eq!(request.runtime.auto_compact_token_limit, Some(999));
8491    }
8492
8493    #[tokio::test]
8494    async fn runtime_tool_allowlist_filters_advertised_tools() {
8495        let request = captured_profile_request(RuntimeConfig {
8496            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8497            default_model: "gpt-5.5".to_string(),
8498            tool_allowlist: vec!["edit".to_string()],
8499            model_profiles: std::collections::HashMap::from([(
8500                "gpt-5.5".to_string(),
8501                test_model_profile("gpt-5.5"),
8502            )]),
8503            ..RuntimeConfig::default()
8504        })
8505        .await;
8506
8507        let tool_names = request
8508            .tools
8509            .iter()
8510            .map(|tool| tool.name.as_str())
8511            .collect::<Vec<_>>();
8512        assert_eq!(tool_names, vec!["edit"]);
8513    }
8514
8515    /// Builds a store-backed runtime turn for a thread created with the given per-thread
8516    /// overrides and returns the captured inference request.
8517    async fn captured_thread_override_request(
8518        runtime: &Arc<Runtime>,
8519        requests: &Arc<StdMutex<Vec<AgentInferenceRequest>>>,
8520        tool_allowlist: Vec<String>,
8521        developer_instructions: Option<String>,
8522        external_tools: Vec<ToolSpec>,
8523    ) -> AgentInferenceRequest {
8524        let thread_id = runtime
8525            .create_thread_with(CreateThreadRequest {
8526                title: Some("Thread overrides".to_string()),
8527                workspace: test_workspace(),
8528                workspace_id: None,
8529                root_id: None,
8530                provider: None,
8531                model: None,
8532                selection_mode: None,
8533                tool_allowlist,
8534                developer_instructions,
8535                external_tools,
8536                runner: None,
8537            })
8538            .await
8539            .unwrap()
8540            .thread_id;
8541        let mut rx = runtime.subscribe_events();
8542        let turn_id = runtime
8543            .start_turn(StartTurnRequest {
8544                thread_id,
8545                message: "hello".to_string(),
8546                images: Vec::new(),
8547                provider_override: None,
8548                model_override: None,
8549                reasoning_override: None,
8550                workspace: test_workspace(),
8551                instructions: crate::default_instructions(),
8552                developer_context: None,
8553                task_ledger_required: false,
8554            })
8555            .await
8556            .unwrap();
8557        tokio::time::timeout(std::time::Duration::from_secs(5), async {
8558            loop {
8559                let envelope = rx.recv().await.unwrap();
8560                if envelope.turn_id.as_deref() != Some(&turn_id) {
8561                    continue;
8562                }
8563                match envelope.event {
8564                    RoderEvent::TurnCompleted(_) => break,
8565                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8566                    _ => {}
8567                }
8568            }
8569        })
8570        .await
8571        .unwrap();
8572        requests.lock().unwrap().pop().expect("captured request")
8573    }
8574
8575    #[tokio::test]
8576    async fn thread_tool_allowlist_filters_only_that_thread() {
8577        let requests = Arc::new(StdMutex::new(Vec::new()));
8578        let thread_root =
8579            std::env::temp_dir().join(format!("roder-thread-allowlist-{}", uuid::Uuid::new_v4()));
8580        let mut builder = ExtensionRegistryBuilder::new();
8581        builder.inference_engine(Arc::new(SwitchCaptureEngine {
8582            requests: requests.clone(),
8583        }));
8584        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8585            base_path: thread_root.clone(),
8586        }));
8587        builder.tool_contributor(Arc::new(ProfileToolContributor));
8588        let runtime = Arc::new(
8589            Runtime::new(
8590                builder.build().unwrap(),
8591                RuntimeConfig {
8592                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8593                    default_model: "gpt-5.5".to_string(),
8594                    model_profiles: std::collections::HashMap::from([(
8595                        "gpt-5.5".to_string(),
8596                        test_model_profile("gpt-5.5"),
8597                    )]),
8598                    ..RuntimeConfig::default()
8599                },
8600            )
8601            .unwrap(),
8602        );
8603
8604        let allowlisted = captured_thread_override_request(
8605            &runtime,
8606            &requests,
8607            vec!["edit".to_string()],
8608            None,
8609            Vec::new(),
8610        )
8611        .await;
8612        let unrestricted =
8613            captured_thread_override_request(&runtime, &requests, Vec::new(), None, Vec::new())
8614                .await;
8615
8616        let allowlisted_names = allowlisted
8617            .tools
8618            .iter()
8619            .map(|tool| tool.name.as_str())
8620            .collect::<Vec<_>>();
8621        assert_eq!(allowlisted_names, vec!["edit"]);
8622        let unrestricted_names = unrestricted
8623            .tools
8624            .iter()
8625            .map(|tool| tool.name.as_str())
8626            .collect::<Vec<_>>();
8627        assert!(unrestricted_names.contains(&"edit"));
8628        assert!(unrestricted_names.len() > 1);
8629
8630        let _ = std::fs::remove_dir_all(thread_root);
8631    }
8632
8633    /// Builds a store-backed runtime whose `RuntimeConfig.tool_allowlist` is `["edit"]`.
8634    fn runtime_with_edit_allowlist(
8635        requests: &Arc<StdMutex<Vec<AgentInferenceRequest>>>,
8636        thread_root: &std::path::Path,
8637    ) -> Arc<Runtime> {
8638        let mut builder = ExtensionRegistryBuilder::new();
8639        builder.inference_engine(Arc::new(SwitchCaptureEngine {
8640            requests: requests.clone(),
8641        }));
8642        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8643            base_path: thread_root.to_path_buf(),
8644        }));
8645        builder.tool_contributor(Arc::new(ProfileToolContributor));
8646        Arc::new(
8647            Runtime::new(
8648                builder.build().unwrap(),
8649                RuntimeConfig {
8650                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8651                    default_model: "gpt-5.5".to_string(),
8652                    tool_allowlist: vec!["edit".to_string()],
8653                    model_profiles: std::collections::HashMap::from([(
8654                        "gpt-5.5".to_string(),
8655                        test_model_profile("gpt-5.5"),
8656                    )]),
8657                    ..RuntimeConfig::default()
8658                },
8659            )
8660            .unwrap(),
8661        )
8662    }
8663
8664    #[tokio::test]
8665    async fn runtime_and_thread_allowlists_intersect() {
8666        let requests = Arc::new(StdMutex::new(Vec::new()));
8667        let thread_root = std::env::temp_dir().join(format!(
8668            "roder-allowlist-intersect-{}",
8669            uuid::Uuid::new_v4()
8670        ));
8671        let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8672
8673        let request = captured_thread_override_request(
8674            &runtime,
8675            &requests,
8676            vec!["edit".to_string(), "write_file".to_string()],
8677            None,
8678            Vec::new(),
8679        )
8680        .await;
8681
8682        // The thread allowlist must not re-enable tools the runtime allowlist bans.
8683        let names = request
8684            .tools
8685            .iter()
8686            .map(|tool| tool.name.as_str())
8687            .collect::<Vec<_>>();
8688        assert_eq!(names, vec!["edit"]);
8689
8690        let _ = std::fs::remove_dir_all(thread_root);
8691    }
8692
8693    #[tokio::test]
8694    async fn route_tool_call_denies_tools_outside_allowlists() {
8695        let requests = Arc::new(StdMutex::new(Vec::new()));
8696        let thread_root =
8697            std::env::temp_dir().join(format!("roder-allowlist-dispatch-{}", uuid::Uuid::new_v4()));
8698        let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8699        let thread_id = runtime
8700            .create_thread_with(CreateThreadRequest {
8701                title: Some("Dispatch allowlist".to_string()),
8702                workspace: test_workspace(),
8703                workspace_id: None,
8704                root_id: None,
8705                provider: None,
8706                model: None,
8707                selection_mode: None,
8708                tool_allowlist: vec!["edit".to_string(), "write_file".to_string()],
8709                developer_instructions: None,
8710                external_tools: Vec::new(),
8711                runner: None,
8712            })
8713            .await
8714            .unwrap()
8715            .thread_id;
8716
8717        // write_file is registered and on the thread allowlist but banned by the runtime allowlist.
8718        let result = runtime
8719            .route_tool_call(
8720                &thread_id,
8721                &"turn-allowlist-dispatch".to_string(),
8722                roder_api::inference::ToolCallCompleted {
8723                    id: "call-1".to_string(),
8724                    name: "write_file".to_string(),
8725                    arguments: r#"{"path":"a.txt","content":"hi"}"#.to_string(),
8726                },
8727                None,
8728                None,
8729            )
8730            .await
8731            .unwrap();
8732
8733        assert!(result.is_error);
8734        assert!(
8735            result
8736                .result
8737                .contains("not permitted by the tool allowlist"),
8738            "unexpected result: {}",
8739            result.result
8740        );
8741
8742        let _ = std::fs::remove_dir_all(thread_root);
8743    }
8744
8745    #[tokio::test]
8746    async fn route_tool_calls_denies_agent_swarm_mixed_with_other_tools() {
8747        let requests = Arc::new(StdMutex::new(Vec::new()));
8748        let thread_root =
8749            std::env::temp_dir().join(format!("roder-swarm-exclusive-{}", uuid::Uuid::new_v4()));
8750        let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8751
8752        // A response that mixes agent_swarm with another tool is denied wholesale:
8753        // both calls get an error result with retry guidance, and no tool runs.
8754        let results = runtime
8755            .route_tool_calls(
8756                &"thread-swarm".to_string(),
8757                &"turn-swarm".to_string(),
8758                vec![
8759                    roder_api::inference::ToolCallCompleted {
8760                        id: "swarm-1".to_string(),
8761                        name: "agent_swarm".to_string(),
8762                        arguments: "{}".to_string(),
8763                    },
8764                    roder_api::inference::ToolCallCompleted {
8765                        id: "read-1".to_string(),
8766                        name: "read_file".to_string(),
8767                        arguments: "{}".to_string(),
8768                    },
8769                ],
8770                true,
8771                None,
8772                None,
8773            )
8774            .await
8775            .unwrap();
8776
8777        assert_eq!(results.len(), 2, "every tool_call_id must get a response");
8778        assert_eq!(results[0].id, "swarm-1");
8779        assert_eq!(results[1].id, "read-1");
8780        for result in &results {
8781            assert!(result.is_error);
8782            assert!(
8783                result.result.contains("only tool call"),
8784                "unexpected result: {}",
8785                result.result
8786            );
8787        }
8788
8789        let _ = std::fs::remove_dir_all(thread_root);
8790    }
8791
8792    #[tokio::test]
8793    async fn route_tool_calls_emits_agent_swarm_started_event() {
8794        let requests = Arc::new(StdMutex::new(Vec::new()));
8795        let thread_root =
8796            std::env::temp_dir().join(format!("roder-swarm-started-{}", uuid::Uuid::new_v4()));
8797        let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8798        let mut events = runtime.subscribe_events();
8799
8800        // A single agent_swarm call (the valid shape) emits AgentSwarmStarted on
8801        // the event bus before dispatch, with the child count parsed from args.
8802        let _ = runtime
8803            .route_tool_calls(
8804                &"thread-swarm".to_string(),
8805                &"turn-swarm".to_string(),
8806                vec![roder_api::inference::ToolCallCompleted {
8807                    id: "swarm-1".to_string(),
8808                    name: "agent_swarm".to_string(),
8809                    arguments: r#"{"description":"x","prompt_template":"Read {{item}}","items":["a.rs","b.rs"]}"#
8810                        .to_string(),
8811                }],
8812                true,
8813                None,
8814                None,
8815            )
8816            .await
8817            .unwrap();
8818
8819        let mut started_child_count = None;
8820        for _ in 0..16 {
8821            let envelope = tokio::time::timeout(std::time::Duration::from_secs(2), events.recv())
8822                .await
8823                .unwrap()
8824                .unwrap();
8825            if let RoderEvent::AgentSwarmStarted(event) = envelope.event {
8826                started_child_count = Some(event.child_count);
8827                assert_eq!(event.tool_id, "swarm-1");
8828                break;
8829            }
8830        }
8831        assert_eq!(started_child_count, Some(2));
8832
8833        let _ = std::fs::remove_dir_all(thread_root);
8834    }
8835
8836    #[test]
8837    fn agent_swarm_child_count_sums_items_and_resumes() {
8838        assert_eq!(
8839            agent_swarm_child_count(r#"{"description":"x","items":["a","b","c"]}"#),
8840            3
8841        );
8842        assert_eq!(
8843            agent_swarm_child_count(
8844                r#"{"description":"x","items":["a"],"resume_agent_ids":{"id1":"continue"}}"#
8845            ),
8846            2
8847        );
8848        // Malformed input is lenient.
8849        assert_eq!(agent_swarm_child_count("not json"), 0);
8850    }
8851
8852    #[test]
8853    fn parse_swarm_counts_reads_summary_with_omitted_buckets() {
8854        let text = "<agent_swarm_result>\n<summary>completed: 2, failed: 1</summary>\n</agent_swarm_result>";
8855        assert_eq!(parse_swarm_counts(text), Some((2, 1, 0)));
8856        let text = "<agent_swarm_result>\n<summary>completed: 0</summary>\n</agent_swarm_result>";
8857        assert_eq!(parse_swarm_counts(text), Some((0, 0, 0)));
8858        // Not a swarm result.
8859        assert_eq!(parse_swarm_counts("just text"), None);
8860    }
8861
8862    /// Signals when inference starts, then waits for `proceed` and fails the stream.
8863    struct SignalledFailureEngine {
8864        started: tokio::sync::mpsc::UnboundedSender<()>,
8865        proceed: Arc<tokio::sync::Notify>,
8866    }
8867
8868    #[async_trait::async_trait]
8869    impl InferenceEngine for SignalledFailureEngine {
8870        fn id(&self) -> String {
8871            roder_api::catalog::PROVIDER_MOCK.to_string()
8872        }
8873
8874        fn capabilities(&self) -> InferenceCapabilities {
8875            InferenceCapabilities::coding_agent_default()
8876        }
8877
8878        async fn list_models(
8879            &self,
8880            _ctx: InferenceProviderContext<'_>,
8881        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
8882            Ok(roder_api::catalog::models_for_provider(
8883                roder_api::catalog::PROVIDER_MOCK,
8884                true,
8885            ))
8886        }
8887
8888        async fn stream_turn(
8889            &self,
8890            _ctx: InferenceTurnContext<'_>,
8891            _request: AgentInferenceRequest,
8892        ) -> anyhow::Result<InferenceEventStream> {
8893            let _ = self.started.send(());
8894            self.proceed.notified().await;
8895            anyhow::bail!("engine failed mid-turn")
8896        }
8897    }
8898
8899    #[tokio::test]
8900    async fn failed_turn_sweeps_pending_external_tool_calls() {
8901        let (started_tx, mut started_rx) = tokio::sync::mpsc::unbounded_channel();
8902        let proceed = Arc::new(tokio::sync::Notify::new());
8903        let mut builder = ExtensionRegistryBuilder::new();
8904        builder.inference_engine(Arc::new(SignalledFailureEngine {
8905            started: started_tx,
8906            proceed: proceed.clone(),
8907        }));
8908        let runtime =
8909            Arc::new(Runtime::new(builder.build().unwrap(), RuntimeConfig::default()).unwrap());
8910        let mut rx = runtime.subscribe_events();
8911        let turn_id = runtime
8912            .start_turn(StartTurnRequest {
8913                thread_id: "thread-sweep".to_string(),
8914                message: "go".to_string(),
8915                images: Vec::new(),
8916                provider_override: None,
8917                model_override: None,
8918                reasoning_override: None,
8919                workspace: test_workspace(),
8920                instructions: InstructionBundle {
8921                    system: None,
8922                    developer: None,
8923                    developer_context: None,
8924                },
8925                developer_context: None,
8926                task_ledger_required: false,
8927            })
8928            .await
8929            .unwrap();
8930        tokio::time::timeout(std::time::Duration::from_secs(5), started_rx.recv())
8931            .await
8932            .unwrap()
8933            .unwrap();
8934
8935        let (tx, _pending_rx) = oneshot::channel();
8936        runtime.pending_external_tool_calls.lock().await.insert(
8937            "exttool-sweep-test".to_string(),
8938            PendingExternalToolCall {
8939                thread_id: "thread-sweep".to_string(),
8940                turn_id: turn_id.clone(),
8941                tool_id: "call-1".to_string(),
8942                tool_name: "acme_lookup".to_string(),
8943                tx,
8944            },
8945        );
8946        proceed.notify_one();
8947
8948        let outcome = tokio::time::timeout(std::time::Duration::from_secs(5), async {
8949            loop {
8950                let envelope = rx.recv().await.unwrap();
8951                if let RoderEvent::ExternalToolCallResolved(event) = envelope.event
8952                    && event.request_id == "exttool-sweep-test"
8953                {
8954                    break event.outcome;
8955                }
8956            }
8957        })
8958        .await
8959        .expect("turn failure must resolve pending external tool calls");
8960        assert_eq!(outcome, ExternalToolCallOutcome::Cancelled);
8961        assert!(runtime.pending_external_tool_calls.lock().await.is_empty());
8962    }
8963
8964    #[tokio::test]
8965    async fn thread_developer_instructions_layer_under_harness_prompt() {
8966        let requests = Arc::new(StdMutex::new(Vec::new()));
8967        let thread_root = std::env::temp_dir().join(format!(
8968            "roder-thread-instructions-{}",
8969            uuid::Uuid::new_v4()
8970        ));
8971        let mut builder = ExtensionRegistryBuilder::new();
8972        builder.inference_engine(Arc::new(SwitchCaptureEngine {
8973            requests: requests.clone(),
8974        }));
8975        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8976            base_path: thread_root.clone(),
8977        }));
8978        builder.tool_contributor(Arc::new(ProfileToolContributor));
8979        let runtime =
8980            Arc::new(Runtime::new(builder.build().unwrap(), RuntimeConfig::default()).unwrap());
8981
8982        let request = captured_thread_override_request(
8983            &runtime,
8984            &requests,
8985            Vec::new(),
8986            Some("You are embedded in a host app.".to_string()),
8987            Vec::new(),
8988        )
8989        .await;
8990
8991        let system = request.instructions.system.expect("system instructions");
8992        assert!(system.starts_with("You are Roder"));
8993        let developer = request
8994            .instructions
8995            .developer
8996            .expect("developer instructions");
8997        assert!(developer.starts_with("You are embedded in a host app."));
8998
8999        let plain =
9000            captured_thread_override_request(&runtime, &requests, Vec::new(), None, Vec::new())
9001                .await;
9002        assert_eq!(plain.instructions.developer, None);
9003
9004        let _ = std::fs::remove_dir_all(thread_root);
9005    }
9006
9007    #[tokio::test]
9008    async fn thread_external_tools_are_advertised_and_shadow_builtins() {
9009        let requests = Arc::new(StdMutex::new(Vec::new()));
9010        let thread_root = std::env::temp_dir().join(format!(
9011            "roder-thread-external-tools-{}",
9012            uuid::Uuid::new_v4()
9013        ));
9014        let mut builder = ExtensionRegistryBuilder::new();
9015        builder.inference_engine(Arc::new(SwitchCaptureEngine {
9016            requests: requests.clone(),
9017        }));
9018        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
9019            base_path: thread_root.clone(),
9020        }));
9021        builder.tool_contributor(Arc::new(ProfileToolContributor));
9022        let runtime =
9023            Arc::new(Runtime::new(builder.build().unwrap(), RuntimeConfig::default()).unwrap());
9024
9025        let external_tools = vec![
9026            ToolSpec {
9027                name: "acme_lookup".to_string(),
9028                description: "Look up Acme workspace state.".to_string(),
9029                parameters: serde_json::json!({
9030                    "type": "object",
9031                    "properties": { "query": { "type": "string" } },
9032                    "required": ["query"]
9033                }),
9034            },
9035            ToolSpec {
9036                name: "edit".to_string(),
9037                description: "Host-managed edit.".to_string(),
9038                parameters: serde_json::json!({ "type": "object" }),
9039            },
9040        ];
9041        let request =
9042            captured_thread_override_request(&runtime, &requests, Vec::new(), None, external_tools)
9043                .await;
9044
9045        let acme = request
9046            .tools
9047            .iter()
9048            .find(|tool| tool.name == "acme_lookup")
9049            .expect("external tool advertised");
9050        assert_eq!(acme.description, "Look up Acme workspace state.");
9051        assert_eq!(acme.parameters["required"][0], "query");
9052        let edits = request
9053            .tools
9054            .iter()
9055            .filter(|tool| tool.name == "edit")
9056            .collect::<Vec<_>>();
9057        assert_eq!(edits.len(), 1, "external edit shadows the builtin");
9058        assert_eq!(edits[0].description, "Host-managed edit.");
9059
9060        let plain =
9061            captured_thread_override_request(&runtime, &requests, Vec::new(), None, Vec::new())
9062                .await;
9063        assert!(plain.tools.iter().all(|tool| tool.name != "acme_lookup"));
9064        let plain_edit = plain
9065            .tools
9066            .iter()
9067            .find(|tool| tool.name == "edit")
9068            .expect("builtin edit advertised on plain thread");
9069        assert_eq!(plain_edit.description, "edit test tool");
9070
9071        let _ = std::fs::remove_dir_all(thread_root);
9072    }
9073
9074    #[tokio::test]
9075    async fn model_switch_injects_summary_and_records_profile_segments() {
9076        let requests = Arc::new(StdMutex::new(Vec::new()));
9077        let thread_root = std::env::temp_dir().join(format!(
9078            "roder-model-switch-thread-{}",
9079            uuid::Uuid::new_v4()
9080        ));
9081        let mut builder = ExtensionRegistryBuilder::new();
9082        builder.inference_engine(Arc::new(SwitchCaptureEngine {
9083            requests: requests.clone(),
9084        }));
9085        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
9086            base_path: thread_root.clone(),
9087        }));
9088        builder.tool_contributor(Arc::new(ProfileToolContributor));
9089        let mut claude_profile = test_model_profile("claude-haiku-4-5-20251001");
9090        claude_profile.provider_family = ProviderFamily::Anthropic;
9091        claude_profile.edit_tool = Some(EDIT_TOOL_EDIT.to_string());
9092        let runtime = Arc::new(
9093            Runtime::new(
9094                builder.build().unwrap(),
9095                RuntimeConfig {
9096                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9097                    default_model: "gpt-5.5".to_string(),
9098                    model_profiles: std::collections::HashMap::from([
9099                        ("gpt-5.5".to_string(), test_model_profile("gpt-5.5")),
9100                        ("claude-haiku-4-5-20251001".to_string(), claude_profile),
9101                    ]),
9102                    ..RuntimeConfig::default()
9103                },
9104            )
9105            .unwrap(),
9106        );
9107        let thread_id = runtime
9108            .create_thread_with(CreateThreadRequest {
9109                title: Some("Model switch".to_string()),
9110                workspace: test_workspace(),
9111                workspace_id: None,
9112                root_id: None,
9113                provider: None,
9114                model: None,
9115                selection_mode: None,
9116                tool_allowlist: Vec::new(),
9117                developer_instructions: None,
9118                external_tools: Vec::new(),
9119                runner: None,
9120            })
9121            .await
9122            .unwrap()
9123            .thread_id;
9124        let mut rx = runtime.subscribe_events();
9125        for (message, model_override) in [
9126            ("first turn", None),
9127            ("second turn", Some("claude-haiku-4-5-20251001".to_string())),
9128        ] {
9129            let turn_id = runtime
9130                .start_turn(StartTurnRequest {
9131                    thread_id: thread_id.clone(),
9132                    message: message.to_string(),
9133                    images: Vec::new(),
9134                    provider_override: None,
9135                    model_override,
9136                    reasoning_override: None,
9137                    workspace: test_workspace(),
9138                    instructions: InstructionBundle::default(),
9139                    developer_context: None,
9140                    task_ledger_required: false,
9141                })
9142                .await
9143                .unwrap();
9144            tokio::time::timeout(std::time::Duration::from_secs(5), async {
9145                loop {
9146                    let envelope = rx.recv().await.unwrap();
9147                    if envelope.turn_id.as_deref() != Some(&turn_id) {
9148                        continue;
9149                    }
9150                    match envelope.event {
9151                        RoderEvent::TurnCompleted(_) => break,
9152                        RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9153                        _ => {}
9154                    }
9155                }
9156            })
9157            .await
9158            .unwrap();
9159        }
9160
9161        let captured = requests.lock().unwrap().clone();
9162        assert_eq!(captured.len(), 2);
9163        assert!(captured[1].transcript.iter().any(|item| {
9164            matches!(
9165                item,
9166                TranscriptItem::UserMessage(message)
9167                    if message.text.starts_with(MODEL_SWITCH_SUMMARY_PREFIX)
9168                        && message.text.contains("previous profile mock/gpt-5.5")
9169                        && message.text.contains("Current profile mock/claude-haiku-4-5-20251001")
9170                        && message.text.contains("Available tools now:")
9171            )
9172        }));
9173
9174        let snapshot = runtime
9175            .thread_store
9176            .as_ref()
9177            .unwrap()
9178            .load_thread(&thread_id)
9179            .await
9180            .unwrap()
9181            .unwrap();
9182        let trace_segments = snapshot
9183            .turns
9184            .iter()
9185            .flat_map(|turn| &turn.items)
9186            .filter(|item| {
9187                matches!(
9188                    item,
9189                    TranscriptItem::ProviderMetadata(value)
9190                        if value.get("kind").and_then(serde_json::Value::as_str)
9191                            == Some(MODEL_PROFILE_TRACE_KIND)
9192                            && value.get("segment").and_then(serde_json::Value::as_str)
9193                                == Some("assistant")
9194                )
9195            })
9196            .count();
9197        assert!(trace_segments >= 2);
9198        let _ = std::fs::remove_dir_all(thread_root);
9199    }
9200
9201    struct CountingTaskTool {
9202        calls: Arc<StdMutex<u32>>,
9203    }
9204
9205    #[async_trait::async_trait]
9206    impl ToolExecutor for CountingTaskTool {
9207        fn spec(&self) -> ToolSpec {
9208            ToolSpec {
9209                name: "task".to_string(),
9210                description: "Dispatch a test subagent.".to_string(),
9211                parameters: serde_json::json!({
9212                    "type": "object",
9213                    "properties": {
9214                        "description": { "type": "string" },
9215                        "prompt": { "type": "string" },
9216                        "parent_deadline_seconds": { "type": "integer" }
9217                    },
9218                    "required": ["description", "prompt"],
9219                    "additionalProperties": false
9220                }),
9221            }
9222        }
9223
9224        async fn execute(
9225            &self,
9226            _ctx: ToolExecutionContext,
9227            call: ToolCall,
9228        ) -> anyhow::Result<ToolResult> {
9229            *self.calls.lock().unwrap() += 1;
9230            Ok(ToolResult {
9231                id: call.id,
9232                name: call.name,
9233                text: "started child".to_string(),
9234                data: serde_json::json!({}),
9235                is_error: false,
9236            })
9237        }
9238    }
9239
9240    #[tokio::test]
9241    async fn runtime_profile_reaches_inference_request_and_turn_metadata() {
9242        let captured = Arc::new(StdMutex::new(None));
9243        let mut builder = ExtensionRegistryBuilder::new();
9244        builder.inference_engine(Arc::new(CapturingEngine {
9245            request: captured.clone(),
9246        }));
9247        let runtime = Arc::new(
9248            Runtime::new(
9249                builder.build().unwrap(),
9250                RuntimeConfig {
9251                    runtime_profile: RuntimeProfile::NonInteractive,
9252                    ..RuntimeConfig::default()
9253                },
9254            )
9255            .unwrap(),
9256        );
9257        let mut rx = runtime.subscribe_events();
9258        let turn_id = runtime
9259            .start_turn(StartTurnRequest {
9260                thread_id: "thread-profile".to_string(),
9261                message: "work unattended".to_string(),
9262                images: Vec::new(),
9263                provider_override: None,
9264                model_override: None,
9265                reasoning_override: None,
9266                workspace: test_workspace(),
9267                instructions: InstructionBundle {
9268                    system: None,
9269                    developer: Some("base developer".to_string()),
9270                    developer_context: None,
9271                },
9272                developer_context: None,
9273                task_ledger_required: false,
9274            })
9275            .await
9276            .unwrap();
9277
9278        let mut observed_profile = None;
9279        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9280            loop {
9281                let envelope = rx.recv().await.unwrap();
9282                if envelope.turn_id.as_deref() != Some(&turn_id) {
9283                    continue;
9284                }
9285                match envelope.event {
9286                    RoderEvent::TurnStarted(event) => {
9287                        observed_profile = Some(event.runtime_profile);
9288                    }
9289                    RoderEvent::TurnCompleted(_) => break,
9290                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9291                    _ => {}
9292                }
9293            }
9294        })
9295        .await
9296        .unwrap();
9297
9298        assert_eq!(observed_profile, Some(RuntimeProfile::NonInteractive));
9299        let request = captured.lock().unwrap().clone().unwrap();
9300        assert_eq!(request.runtime.profile, RuntimeProfile::NonInteractive);
9301        let developer = request.instructions.developer.unwrap();
9302        assert!(developer.contains("base developer"));
9303        assert!(developer.contains("non-interactive profile"));
9304    }
9305
9306    #[tokio::test]
9307    async fn global_policy_mode_changes_do_not_create_runtime_thread_directory() {
9308        let workspace = runtime_test_workspace("global-policy-mode");
9309        let thread_root = workspace.join("threads");
9310        let mut builder = ExtensionRegistryBuilder::new();
9311        builder.inference_engine(Arc::new(FakeInferenceEngine));
9312        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
9313            base_path: thread_root.clone(),
9314        }));
9315        let runtime = Runtime::new(
9316            builder.build().unwrap(),
9317            RuntimeConfig {
9318                workspace: Some(workspace.display().to_string()),
9319                ..Default::default()
9320            },
9321        )
9322        .unwrap();
9323
9324        runtime
9325            .set_policy_mode(PolicyMode::AcceptAll, Some("test".to_string()))
9326            .await
9327            .unwrap();
9328
9329        assert!(!thread_root.join("runtime").exists());
9330        let _ = std::fs::remove_dir_all(workspace);
9331    }
9332
9333    #[tokio::test]
9334    async fn task_ledger_enforcement_injects_eval_reminder_before_work() {
9335        let captured = Arc::new(StdMutex::new(None));
9336        let mut builder = ExtensionRegistryBuilder::new();
9337        builder.inference_engine(Arc::new(CapturingEngine {
9338            request: captured.clone(),
9339        }));
9340        builder.tool_contributor(Arc::new(
9341            roder_ext_task_ledger::TaskLedgerToolContributor::default(),
9342        ));
9343        let runtime = Arc::new(
9344            Runtime::new(
9345                builder.build().unwrap(),
9346                RuntimeConfig {
9347                    runtime_profile: RuntimeProfile::Eval,
9348                    policy_mode: PolicyMode::Bypass,
9349                    agent_swarm_mode: false,
9350                    ultra_mode: false,
9351                    ..RuntimeConfig::default()
9352                },
9353            )
9354            .unwrap(),
9355        );
9356        let mut rx = runtime.subscribe_events();
9357        let turn_id = runtime
9358            .start_turn(StartTurnRequest {
9359                thread_id: "thread-ledger".to_string(),
9360                message: "decomposed work".to_string(),
9361                images: Vec::new(),
9362                provider_override: None,
9363                model_override: None,
9364                reasoning_override: None,
9365                workspace: test_workspace(),
9366                instructions: InstructionBundle::default(),
9367                developer_context: None,
9368                task_ledger_required: true,
9369            })
9370            .await
9371            .unwrap();
9372
9373        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9374            loop {
9375                let envelope = rx.recv().await.unwrap();
9376                if envelope.turn_id.as_deref() == Some(&turn_id)
9377                    && matches!(envelope.event, RoderEvent::TurnCompleted(_))
9378                {
9379                    break;
9380                }
9381            }
9382        })
9383        .await
9384        .unwrap();
9385
9386        let request = captured.lock().unwrap().clone().unwrap();
9387        let developer = request.instructions.developer.unwrap();
9388        assert!(developer.contains("Task Ledger Required"));
9389        assert!(developer.contains("task_ledger.update"));
9390        let tool_names: Vec<_> = request
9391            .tools
9392            .iter()
9393            .map(|tool| tool.name.as_str())
9394            .collect();
9395        assert!(
9396            tool_names.contains(&TASK_LEDGER_TOOL_NAME),
9397            "tool names: {tool_names:?}"
9398        );
9399        assert_eq!(
9400            request.tool_choice,
9401            ToolChoice::Specific(TASK_LEDGER_TOOL_NAME.to_string())
9402        );
9403        assert_eq!(request.tools.len(), 1);
9404        assert_eq!(request.tools[0].name, TASK_LEDGER_TOOL_NAME);
9405    }
9406
9407    #[tokio::test]
9408    async fn eval_task_ledger_blocks_final_answer_until_open_items_are_completed() {
9409        let requests = Arc::new(StdMutex::new(Vec::new()));
9410        let mut builder = ExtensionRegistryBuilder::new();
9411        builder.inference_engine(Arc::new(TaskLedgerCompletionGateEngine {
9412            calls: StdMutex::new(0),
9413            requests: requests.clone(),
9414        }));
9415        builder.tool_contributor(Arc::new(
9416            roder_ext_task_ledger::TaskLedgerToolContributor::default(),
9417        ));
9418        let runtime = Arc::new(
9419            Runtime::new(
9420                builder.build().unwrap(),
9421                RuntimeConfig {
9422                    runtime_profile: RuntimeProfile::Eval,
9423                    policy_mode: PolicyMode::Bypass,
9424                    agent_swarm_mode: false,
9425                    ultra_mode: false,
9426                    ..RuntimeConfig::default()
9427                },
9428            )
9429            .unwrap(),
9430        );
9431        let mut rx = runtime.subscribe_events();
9432        let turn_id = runtime
9433            .start_turn(StartTurnRequest {
9434                thread_id: "thread-ledger-completion".to_string(),
9435                message: "write the answer file".to_string(),
9436                images: Vec::new(),
9437                provider_override: None,
9438                model_override: None,
9439                reasoning_override: None,
9440                workspace: test_workspace(),
9441                instructions: InstructionBundle::default(),
9442                developer_context: None,
9443                task_ledger_required: true,
9444            })
9445            .await
9446            .unwrap();
9447
9448        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9449            loop {
9450                let envelope = rx.recv().await.unwrap();
9451                if envelope.turn_id.as_deref() != Some(&turn_id) {
9452                    continue;
9453                }
9454                match envelope.event {
9455                    RoderEvent::TurnCompleted(_) => break,
9456                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9457                    _ => {}
9458                }
9459            }
9460        })
9461        .await
9462        .unwrap();
9463
9464        let requests = requests.lock().unwrap().clone();
9465        assert_eq!(requests.len(), 4);
9466        assert!(requests[2].transcript.iter().any(|item| {
9467            matches!(
9468                item,
9469                TranscriptItem::UserMessage(message)
9470                    if message.text.contains("Task Ledger Completion Required")
9471                        && message.text.contains("Write /app/result.txt")
9472            )
9473        }));
9474        assert!(requests[3].transcript.iter().any(|item| {
9475            matches!(
9476                item,
9477                TranscriptItem::ToolResult(result)
9478                    if result.name.as_deref() == Some(TASK_LEDGER_TOOL_NAME)
9479                        && result.result.contains("Task ledger: 2/2 completed")
9480            )
9481        }));
9482    }
9483
9484    #[tokio::test]
9485    async fn eval_task_ledger_checkpoint_requests_scoreable_file_before_final_reserve() {
9486        let requests = Arc::new(StdMutex::new(Vec::new()));
9487        let mut builder = ExtensionRegistryBuilder::new();
9488        builder.inference_engine(Arc::new(TaskLedgerCompletionGateEngine {
9489            calls: StdMutex::new(0),
9490            requests: requests.clone(),
9491        }));
9492        builder.tool_contributor(Arc::new(
9493            roder_ext_task_ledger::TaskLedgerToolContributor::default(),
9494        ));
9495        let runtime = Arc::new(
9496            Runtime::new(
9497                builder.build().unwrap(),
9498                RuntimeConfig {
9499                    runtime_profile: RuntimeProfile::Eval,
9500                    policy_mode: PolicyMode::Bypass,
9501                    agent_swarm_mode: false,
9502                    ultra_mode: false,
9503                    turn_deadline_seconds: Some(120),
9504                    ..RuntimeConfig::default()
9505                },
9506            )
9507            .unwrap(),
9508        );
9509        let mut rx = runtime.subscribe_events();
9510        let turn_id = runtime
9511            .start_turn(StartTurnRequest {
9512                thread_id: "thread-ledger-checkpoint".to_string(),
9513                message: "write the answer file".to_string(),
9514                images: Vec::new(),
9515                provider_override: None,
9516                model_override: None,
9517                reasoning_override: None,
9518                workspace: test_workspace(),
9519                instructions: InstructionBundle::default(),
9520                developer_context: None,
9521                task_ledger_required: true,
9522            })
9523            .await
9524            .unwrap();
9525
9526        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9527            loop {
9528                let envelope = rx.recv().await.unwrap();
9529                if envelope.turn_id.as_deref() != Some(&turn_id) {
9530                    continue;
9531                }
9532                match envelope.event {
9533                    RoderEvent::TurnCompleted(_) => break,
9534                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9535                    _ => {}
9536                }
9537            }
9538        })
9539        .await
9540        .unwrap();
9541
9542        let requests = requests.lock().unwrap().clone();
9543        assert!(requests.len() >= 2);
9544        assert!(requests[1].transcript.iter().any(|item| {
9545            matches!(
9546                item,
9547                TranscriptItem::UserMessage(message)
9548                    if message.text.contains("Scoreable Output Checkpoint")
9549                        && message.text.contains("ensure the required output file(s) exist")
9550                        && message.text.contains("Write /app/result.txt")
9551            )
9552        }));
9553    }
9554
9555    #[test]
9556    fn deadline_task_ledger_prompt_preserves_scoreable_work_instruction() {
9557        let prompt = task_ledger_deadline_completion_prompt(
9558            12,
9559            30,
9560            "Task Ledger Completion Required: write /app/result.txt, then call task_ledger.update",
9561        );
9562
9563        assert!(prompt.contains("12 seconds remain"));
9564        assert!(prompt.contains("create or update the required scoreable output files"));
9565        assert!(prompt.contains("write /app/result.txt"));
9566        assert!(prompt.contains(TASK_LEDGER_TOOL_NAME));
9567    }
9568
9569    #[test]
9570    fn scoreable_checkpoint_prompt_preserves_provisional_file_instruction() {
9571        let prompt = task_ledger_scoreable_checkpoint_prompt(
9572            120,
9573            "Task Ledger Completion Required: write /app/result.txt, then call task_ledger.update",
9574        );
9575
9576        assert!(prompt.contains("120 seconds remain"));
9577        assert!(prompt.contains("best evidence-backed answer"));
9578        assert!(prompt.contains("even if provisional"));
9579        assert!(prompt.contains("preserve that candidate"));
9580        assert!(prompt.contains("partial-coverage"));
9581        assert!(prompt.contains("write /app/result.txt"));
9582        assert!(prompt.contains(TASK_LEDGER_TOOL_NAME));
9583    }
9584
9585    #[test]
9586    fn open_task_ledger_moves_inference_timeout_to_scoreable_checkpoint() {
9587        let deadline = Some(OffsetDateTime::now_utc() + Duration::seconds(870));
9588        let transcript = vec![TranscriptItem::ToolResult(ToolResultRecord {
9589            id: "ledger-open".to_string(),
9590            name: Some(TASK_LEDGER_TOOL_NAME.to_string()),
9591            result: "Task ledger: 0/1 completed\n- pending: Write /app/result.txt [write]"
9592                .to_string(),
9593            display_payload: None,
9594            is_error: false,
9595        })];
9596
9597        let (_, action) = inference_timeout_deadline(
9598            deadline,
9599            RuntimeProfile::Eval,
9600            true,
9601            30,
9602            false,
9603            0,
9604            &transcript,
9605        )
9606        .unwrap();
9607
9608        assert_eq!(action, InferenceTimeoutAction::ScoreableCheckpoint);
9609    }
9610
9611    #[tokio::test]
9612    async fn verification_gate_forces_eval_code_changes_through_review() {
9613        let mut builder = ExtensionRegistryBuilder::new();
9614        builder.inference_engine(Arc::new(VerificationGateEngine {
9615            calls: StdMutex::new(0),
9616        }));
9617        builder.tool_contributor(Arc::new(WriteFileContributor));
9618        builder.tool_contributor(Arc::new(
9619            roder_ext_verification::VerificationToolContributor,
9620        ));
9621        let runtime = Arc::new(
9622            Runtime::new(
9623                builder.build().unwrap(),
9624                RuntimeConfig {
9625                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9626                    default_model: "mock".to_string(),
9627                    runtime_profile: RuntimeProfile::Eval,
9628                    policy_mode: PolicyMode::Bypass,
9629                    agent_swarm_mode: false,
9630                    ultra_mode: false,
9631                    ..RuntimeConfig::default()
9632                },
9633            )
9634            .unwrap(),
9635        );
9636        let mut rx = runtime.subscribe_events();
9637        let turn_id = runtime
9638            .start_turn(StartTurnRequest {
9639                thread_id: "thread-verification".to_string(),
9640                message: "write code".to_string(),
9641                images: Vec::new(),
9642                provider_override: None,
9643                model_override: None,
9644                reasoning_override: None,
9645                workspace: test_workspace(),
9646                instructions: InstructionBundle::default(),
9647                developer_context: None,
9648                task_ledger_required: false,
9649            })
9650            .await
9651            .unwrap();
9652
9653        let mut saw_required = false;
9654        let mut saw_completed = false;
9655        let mut final_text = String::new();
9656        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9657            loop {
9658                let envelope = rx.recv().await.unwrap();
9659                if envelope.turn_id.as_deref() != Some(&turn_id) {
9660                    continue;
9661                }
9662                match envelope.event {
9663                    RoderEvent::VerificationRequired(event) => {
9664                        saw_required = true;
9665                        assert_eq!(event.changed_files, vec!["src/lib.rs"]);
9666                    }
9667                    RoderEvent::VerificationCompleted(event) => {
9668                        saw_completed = true;
9669                        assert!(event.passed);
9670                    }
9671                    RoderEvent::InferenceEventReceived(event) => {
9672                        if let InferenceEvent::MessageDelta(delta) = event.event {
9673                            final_text.push_str(&delta.text);
9674                        }
9675                    }
9676                    RoderEvent::TurnCompleted(_) => break,
9677                    _ => {}
9678                }
9679            }
9680        })
9681        .await
9682        .unwrap();
9683
9684        assert!(saw_required);
9685        assert!(saw_completed);
9686        assert!(final_text.contains("verified final"));
9687    }
9688
9689    #[tokio::test]
9690    async fn speed_policy_changes_reasoning_across_eval_model_calls_without_model_switch() {
9691        let requests = Arc::new(StdMutex::new(Vec::new()));
9692        let mut builder = ExtensionRegistryBuilder::new();
9693        builder.inference_engine(Arc::new(SpeedPolicyEngine {
9694            calls: StdMutex::new(0),
9695            requests: requests.clone(),
9696        }));
9697        builder.tool_contributor(Arc::new(WriteFileContributor));
9698        builder.tool_contributor(Arc::new(
9699            roder_ext_verification::VerificationToolContributor,
9700        ));
9701        let runtime = Arc::new(
9702            Runtime::new(
9703                builder.build().unwrap(),
9704                RuntimeConfig {
9705                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9706                    default_model: "gpt-5.5".to_string(),
9707                    runtime_profile: RuntimeProfile::Eval,
9708                    policy_mode: PolicyMode::Bypass,
9709                    agent_swarm_mode: false,
9710                    ultra_mode: false,
9711                    ..RuntimeConfig::default()
9712                },
9713            )
9714            .unwrap(),
9715        );
9716        let mut rx = runtime.subscribe_events();
9717        let turn_id = runtime
9718            .start_turn(StartTurnRequest {
9719                thread_id: "thread-speed-policy".to_string(),
9720                message: "write code".to_string(),
9721                images: Vec::new(),
9722                provider_override: None,
9723                model_override: None,
9724                reasoning_override: None,
9725                workspace: test_workspace(),
9726                instructions: InstructionBundle::default(),
9727                developer_context: None,
9728                task_ledger_required: false,
9729            })
9730            .await
9731            .unwrap();
9732
9733        let mut saw_speed_policy_event = false;
9734        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9735            loop {
9736                let envelope = rx.recv().await.unwrap();
9737                if envelope.turn_id.as_deref() != Some(&turn_id) {
9738                    continue;
9739                }
9740                match envelope.event {
9741                    RoderEvent::InferenceStarted(event) => {
9742                        if event.speed_policy.is_some() {
9743                            saw_speed_policy_event = true;
9744                        }
9745                    }
9746                    RoderEvent::TurnCompleted(_) => break,
9747                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9748                    _ => {}
9749                }
9750            }
9751        })
9752        .await
9753        .unwrap();
9754
9755        let requests = requests.lock().unwrap().clone();
9756        assert!(saw_speed_policy_event);
9757        assert!(requests.len() >= 4);
9758        assert!(requests.iter().all(|request| {
9759            request.model.provider == roder_api::catalog::PROVIDER_MOCK
9760                && request.model.model == "gpt-5.5"
9761        }));
9762        assert_eq!(
9763            requests[0].runtime.speed_policy.as_ref().map(|d| d.phase),
9764            Some(roder_api::inference::SpeedPolicyPhase::Orientation)
9765        );
9766        assert_eq!(requests[0].reasoning.level.as_deref(), Some(REASONING_HIGH));
9767        assert_eq!(
9768            requests[1].runtime.speed_policy.as_ref().map(|d| d.phase),
9769            Some(roder_api::inference::SpeedPolicyPhase::Execution)
9770        );
9771        assert_eq!(requests[1].reasoning.level.as_deref(), Some(REASONING_LOW));
9772        assert_eq!(
9773            requests[2].runtime.speed_policy.as_ref().map(|d| d.phase),
9774            Some(roder_api::inference::SpeedPolicyPhase::Verification)
9775        );
9776        assert_eq!(requests[2].reasoning.level.as_deref(), Some(REASONING_HIGH));
9777        assert_eq!(
9778            requests[2]
9779                .metadata
9780                .pointer("/speedPolicy/phase")
9781                .and_then(serde_json::Value::as_str),
9782            Some("verification")
9783        );
9784    }
9785
9786    #[tokio::test]
9787    async fn deadline_turn_timeout_emits_partial_result_and_clears_active_turn() {
9788        let mut builder = ExtensionRegistryBuilder::new();
9789        builder.inference_engine(Arc::new(DeadlineEngine));
9790        let runtime = Arc::new(
9791            Runtime::new(
9792                builder.build().unwrap(),
9793                RuntimeConfig {
9794                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9795                    default_model: "mock".to_string(),
9796                    runtime_profile: RuntimeProfile::Eval,
9797                    turn_deadline_seconds: Some(1),
9798                    ..RuntimeConfig::default()
9799                },
9800            )
9801            .unwrap(),
9802        );
9803        let mut rx = runtime.subscribe_events();
9804        let turn_id = runtime
9805            .start_turn(StartTurnRequest {
9806                thread_id: "thread-deadline".to_string(),
9807                message: "slow work".to_string(),
9808                images: Vec::new(),
9809                provider_override: None,
9810                model_override: None,
9811                reasoning_override: None,
9812                workspace: test_workspace(),
9813                instructions: InstructionBundle::default(),
9814                developer_context: None,
9815                task_ledger_required: false,
9816            })
9817            .await
9818            .unwrap();
9819
9820        let mut saw_partial = false;
9821        let mut saw_deadline = false;
9822        let mut failed_kind = None;
9823        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9824            loop {
9825                let envelope = rx.recv().await.unwrap();
9826                if envelope.turn_id.as_deref() != Some(&turn_id) {
9827                    continue;
9828                }
9829                match envelope.event {
9830                    RoderEvent::TurnPartialResult(event) => {
9831                        saw_partial = event.summary.contains("partial turn state");
9832                    }
9833                    RoderEvent::TurnDeadlineExceeded(event) => {
9834                        saw_deadline = event.partial_result.contains("transcript items");
9835                    }
9836                    RoderEvent::TurnFailed(event) => {
9837                        failed_kind = event.error_kind;
9838                        break;
9839                    }
9840                    _ => {}
9841                }
9842            }
9843        })
9844        .await
9845        .unwrap();
9846
9847        for _ in 0..20 {
9848            if !runtime.active_turns.read().await.contains_key(&turn_id) {
9849                break;
9850            }
9851            tokio::time::sleep(std::time::Duration::from_millis(10)).await;
9852        }
9853        assert!(saw_partial);
9854        assert!(saw_deadline);
9855        assert_eq!(failed_kind.as_deref(), Some("deadline_timeout"));
9856        assert!(!runtime.active_turns.read().await.contains_key(&turn_id));
9857    }
9858
9859    #[tokio::test]
9860    async fn deadline_skips_subagent_task_when_remaining_budget_is_too_low() {
9861        let calls = Arc::new(StdMutex::new(0));
9862        let mut builder = ExtensionRegistryBuilder::new();
9863        builder.inference_engine(Arc::new(CapturingEngine {
9864            request: Arc::new(StdMutex::new(None)),
9865        }));
9866        let task_tool = Arc::new(CountingTaskTool {
9867            calls: calls.clone(),
9868        });
9869        builder.tool_contributor(Arc::new(TestToolContributor { tool: task_tool }));
9870        let runtime = Arc::new(
9871            Runtime::new(
9872                builder.build().unwrap(),
9873                RuntimeConfig {
9874                    policy_mode: PolicyMode::Bypass,
9875                    agent_swarm_mode: false,
9876                    ultra_mode: false,
9877                    ..RuntimeConfig::default()
9878                },
9879            )
9880            .unwrap(),
9881        );
9882
9883        let result = runtime
9884            .route_tool_call(
9885                &"thread-deadline-task".to_string(),
9886                &"turn-deadline-task".to_string(),
9887                ToolCallCompleted {
9888                    id: "task-1".to_string(),
9889                    name: "task".to_string(),
9890                    arguments: serde_json::json!({
9891                        "description": "inspect",
9892                        "prompt": "read"
9893                    })
9894                    .to_string(),
9895                },
9896                None,
9897                Some(OffsetDateTime::now_utc() + Duration::seconds(1)),
9898            )
9899            .await
9900            .unwrap();
9901
9902        assert!(result.is_error);
9903        assert!(result.result.contains("deadline policy skipped"));
9904        assert_eq!(*calls.lock().unwrap(), 0);
9905    }
9906
9907    struct TestToolContributor {
9908        tool: Arc<dyn ToolExecutor>,
9909    }
9910
9911    impl ToolContributor for TestToolContributor {
9912        fn id(&self) -> String {
9913            "test-tool".to_string()
9914        }
9915
9916        fn contribute(&self, registry: &mut ToolRegistry) -> anyhow::Result<()> {
9917            registry.register(self.tool.clone())
9918        }
9919    }
9920
9921    #[tokio::test]
9922    async fn agent_swarm_mode_override_is_per_thread() {
9923        let runtime = Runtime::fake().unwrap();
9924        let trigger = roder_api::subagents::AgentSwarmModeTrigger::Manual;
9925
9926        // Off everywhere by default.
9927        assert!(
9928            !runtime
9929                .effective_agent_swarm_mode_for_thread("thread-a")
9930                .await
9931        );
9932        assert!(
9933            !runtime
9934                .effective_agent_swarm_mode_for_thread("thread-b")
9935                .await
9936        );
9937
9938        // Enabling on thread-a does not leak into thread-b.
9939        assert!(
9940            runtime
9941                .set_agent_swarm_mode_for_thread("thread-a", true, trigger)
9942                .await
9943        );
9944        assert!(
9945            runtime
9946                .effective_agent_swarm_mode_for_thread("thread-a")
9947                .await
9948        );
9949        assert!(
9950            !runtime
9951                .effective_agent_swarm_mode_for_thread("thread-b")
9952                .await
9953        );
9954
9955        // A per-thread `off` override wins over a runtime-global `on` default.
9956        runtime.set_agent_swarm_mode(true, trigger).await.unwrap();
9957        runtime
9958            .set_agent_swarm_mode_for_thread("thread-b", false, trigger)
9959            .await;
9960        assert!(
9961            !runtime
9962                .effective_agent_swarm_mode_for_thread("thread-b")
9963                .await,
9964            "explicit per-thread off overrides the global on default"
9965        );
9966        // A thread with no override still follows the global default.
9967        assert!(
9968            runtime
9969                .effective_agent_swarm_mode_for_thread("thread-c")
9970                .await,
9971            "threads without an override follow the runtime-global default"
9972        );
9973    }
9974
9975    #[tokio::test]
9976    async fn set_agent_swarm_mode_for_thread_emits_event_with_real_thread_id() {
9977        let runtime = Runtime::fake().unwrap();
9978        let mut events = runtime.subscribe_events();
9979        runtime
9980            .set_agent_swarm_mode_for_thread(
9981                "thread-xyz",
9982                true,
9983                roder_api::subagents::AgentSwarmModeTrigger::Task,
9984            )
9985            .await;
9986        let mut saw = false;
9987        for _ in 0..8 {
9988            let envelope = tokio::time::timeout(std::time::Duration::from_secs(2), events.recv())
9989                .await
9990                .unwrap()
9991                .unwrap();
9992            if let RoderEvent::AgentSwarmModeChanged(event) = envelope.event {
9993                assert_eq!(event.thread_id, "thread-xyz");
9994                assert!(event.enabled);
9995                assert_eq!(
9996                    event.trigger,
9997                    roder_api::subagents::AgentSwarmModeTrigger::Task
9998                );
9999                saw = true;
10000                break;
10001            }
10002        }
10003        assert!(
10004            saw,
10005            "expected an AgentSwarmModeChanged event for the thread"
10006        );
10007    }
10008
10009    #[tokio::test]
10010    async fn ultra_mode_override_is_per_thread() {
10011        let runtime = Runtime::fake().unwrap();
10012        let trigger = roder_api::subagents::UltraModeTrigger::Manual;
10013
10014        assert!(!runtime.effective_ultra_mode_for_thread("thread-a").await);
10015        assert!(!runtime.effective_ultra_mode_for_thread("thread-b").await);
10016
10017        assert!(
10018            runtime
10019                .set_ultra_mode_for_thread("thread-a", true, trigger)
10020                .await
10021        );
10022        assert!(runtime.effective_ultra_mode_for_thread("thread-a").await);
10023        assert!(!runtime.effective_ultra_mode_for_thread("thread-b").await);
10024
10025        runtime.set_ultra_mode(true, trigger).await.unwrap();
10026        runtime
10027            .set_ultra_mode_for_thread("thread-b", false, trigger)
10028            .await;
10029        assert!(
10030            !runtime.effective_ultra_mode_for_thread("thread-b").await,
10031            "explicit per-thread off overrides the global on default"
10032        );
10033        assert!(
10034            runtime.effective_ultra_mode_for_thread("thread-c").await,
10035            "threads without an override follow the runtime-global default"
10036        );
10037    }
10038
10039    #[tokio::test]
10040    async fn set_ultra_mode_for_thread_emits_event_with_real_thread_id() {
10041        let runtime = Runtime::fake().unwrap();
10042        let mut events = runtime.subscribe_events();
10043        runtime
10044            .set_ultra_mode_for_thread(
10045                "thread-ultra",
10046                true,
10047                roder_api::subagents::UltraModeTrigger::Task,
10048            )
10049            .await;
10050        let mut saw = false;
10051        for _ in 0..8 {
10052            let envelope = tokio::time::timeout(std::time::Duration::from_secs(2), events.recv())
10053                .await
10054                .unwrap()
10055                .unwrap();
10056            if let RoderEvent::UltraModeChanged(event) = envelope.event {
10057                assert_eq!(event.thread_id, "thread-ultra");
10058                assert!(event.enabled);
10059                assert_eq!(event.trigger, roder_api::subagents::UltraModeTrigger::Task);
10060                saw = true;
10061                break;
10062            }
10063        }
10064        assert!(saw, "expected an UltraModeChanged event for the thread");
10065    }
10066}