Skip to main content

roder_core/
runtime.rs

1use std::collections::{HashMap, HashSet};
2use std::path::PathBuf;
3use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
4use std::sync::{Arc, Weak};
5
6use anyhow::Context;
7use futures::StreamExt;
8use futures::future::{AbortHandle, Abortable, BoxFuture, try_join_all};
9use roder_api::catalog::{
10    EDIT_TOOL_EDIT, EDIT_TOOL_PATCH, PROVIDER_GEMINI, REASONING_NONE, REASONING_ULTRA,
11    built_in_model_profile, built_in_model_profile_for_provider, lookup_model,
12    model_supports_reasoning_effort,
13};
14use roder_api::context::PolicyGate;
15use roder_api::events::*;
16use roder_api::extension::ExtensionRegistry;
17use roder_api::inference::{
18    AgentInferenceRequest, HostedWebSearchConfig, HostedWebSearchMode, InferenceEngine,
19    InferenceEvent, InferenceTurnContext, InstructionBundle, ModelHarnessProfile,
20    ModelSchemaPolicy, ModelSelection, OutputConfig, ProviderTurnCleanup, ReasoningConfig,
21    RuntimeHints, RuntimeProfile, TokenUsage, ToolCallCompleted, ToolSearchConfig,
22    ToolSearchConfigOverlay, finish_reason_from_stop_reason,
23};
24use roder_api::inference_routing::{InferenceRoutingOutcome, ModelSelectionMode};
25use roder_api::lifecycle::{
26    LifecycleMetricsSnapshot, TurnCleanupOwnership, TurnCleanupState, TurnLifecycleReason,
27    TurnLifecycleRecord, TurnLifecycleSnapshot, TurnLifecycleState, turn_lifecycle_snapshot,
28};
29use roder_api::policy_mode::{PolicyDecision, PolicyMode};
30use roder_api::reliability::{
31    ReliabilityContext, ReliabilityDetails, ReliabilityErrorClass, ReliabilityLimitDecision,
32    ReliabilityLimitRecorded, ReliabilityRequestPolicy, ReliabilityRetryDecision,
33    ReliabilityRetryRecorded, provider_retry_delay_ms,
34};
35use roder_api::remote_runner::{
36    RemoteRunnerProvider, RemoteRunnerSession, RemoteWorkspace, RunnerDestination,
37    RunnerSessionState, ThreadRunnerBinding,
38};
39use roder_api::subagents::SubagentDefinition;
40use roder_api::teams::{
41    TeamId, TeamMailboxMessage, TeamMailboxMessageKind, TeamMemberDescriptor, TeamMemberRole,
42    TeamMemberStatus,
43};
44use roder_api::thread::{
45    ThreadItemEvent, ThreadItemEventKind, ThreadMetadata, ThreadSnapshot, ThreadStore,
46    ThreadUsageMetadata, is_synthetic_event_thread_id, validate_thread_workspace,
47};
48use roder_api::tools::{ToolCall, ToolChoice, ToolExecutionContext, ToolRegistry, ToolResult};
49use roder_api::transcript::{
50    AssistantMessage, ErrorRecord, InputImage, ReasoningSummary, ToolCallRecord, ToolResultRecord,
51    TranscriptItem, UserMessage,
52};
53use roder_sandbox::ScopedFilesystem;
54use roder_sandbox::process::LocalProcessRunner;
55use roder_skills::{SkillRegistry, SkillRegistryOptions};
56use time::{Duration, OffsetDateTime};
57use tokio::sync::{Mutex, Notify, RwLock, oneshot};
58
59mod codex_v2;
60
61use crate::artifacts::{
62    ContextArtifactStore as FilesystemContextArtifactStore, default_context_artifact_dir,
63};
64use crate::bus::EventBus;
65use crate::dynamic_workflows::{
66    DynamicWorkflowEffortProfile, RuntimeDynamicWorkflowConfig, WorkflowTriggerDecision,
67    classify_workflow_trigger, ultracode_reasoning_level_for_model,
68};
69use crate::fake_provider::FakeInferenceEngine;
70use crate::goals::RuntimeGoalController;
71use crate::inference_routing::{
72    InferenceRoutingRequest, RuntimeInferenceRouterConfig, collect_inference_routing_candidates,
73    route_inference_selection, transcript_failure_count_since,
74};
75use crate::instructions::{
76    apply_agent_swarm_mode, apply_codex_multi_agent_mode, apply_model_instruction_overlay,
77    apply_parallel_web_tools, apply_plan_mode, apply_runtime_profile, apply_task_ledger_required,
78    apply_thread_developer_instructions, apply_turn_developer_context,
79};
80use crate::policy_gate::DefaultPolicyGate;
81use crate::reliability::{
82    ReliabilityLimitHit, RuntimeReliabilityConfig, TurnReliabilityState,
83    provider_stream_retry_cause,
84};
85pub use crate::speed_policy::RuntimeSpeedPolicyConfig;
86use crate::speed_policy::{SpeedPolicyState, reasoning_from_decision};
87use crate::subagent_traces::{RuntimeAgentSwarmProgressSink, RuntimeSubagentTraceSink};
88use crate::teams::{TeamManager, TeamMemberStartRequest, TeamStartRequest, TeamState};
89use crate::thread_item_cache::{ThreadItemCache, ThreadItemCacheEntry};
90use crate::verification_gate::VerificationGateState;
91
92const MAX_TOOL_ROUNDS_PER_TURN: usize = 1024;
93/// Injected when `continue_on_failure_limit` turns a consecutive-tool-failure
94/// limit into a continuation, or as the empty-tool-call persistence nudge, so
95/// the model keeps working instead of ending the turn early.
96const RELIABILITY_CONTINUATION_PROMPT: &str = "Verify the task is fully complete. If any part remains unfinished or unverified, keep working using the available tools — try an alternative approach if one is failing. Only stop once the solution is complete and verified; if it is already complete, restate your final answer.";
97/// Capacity of the runtime event broadcast ring buffer. The TUI drains this on
98/// its render loop, which during an active turn only wakes at the ~6 FPS status
99/// animation cadence (backend events do not wake the input poll), so up to
100/// ~166ms of events buffer between drains. A reasoning-high provider that runs
101/// its whole tool loop inside one turn (e.g. claude-code) streams a firehose of
102/// `InferenceEventReceived` deltas plus per-step tool/thinking events; the old
103/// 1024-slot buffer overflowed in those windows and silently dropped tool and
104/// thinking rows from the live view. Sized for generous headroom across bursts
105/// and brief render stalls.
106const EVENT_BUS_CAPACITY: usize = 16_384;
107
108/// Turn id reported to session-scoped local hooks. `SessionStart` and
109/// `SessionEnd` bracket the session rather than any one turn, but the hook
110/// payload carries a turn id, so they share this synthetic value.
111const SESSION_HOOK_TURN_ID: &str = "session";
112pub(crate) const FINAL_ANSWER_PHASE: &str = "final_answer";
113pub(crate) const TASK_LEDGER_TOOL_NAME: &str = "task_ledger.update";
114const TASK_LEDGER_COMPLETION_REMINDER_LIMIT: u8 = 2;
115const TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS: u64 = 180;
116const TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT: u8 = 1;
117pub(crate) const MIN_CHILD_DEADLINE_SECONDS: u64 = 2;
118const MODEL_PROFILE_TRACE_KIND: &str = "model_profile_segment";
119const MODEL_SWITCH_SUMMARY_PREFIX: &str = "Model switch summary:";
120const PROVIDER_CLEANUP_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5);
121
122#[derive(Clone, Copy, Debug, Eq, PartialEq)]
123enum InferenceTimeoutAction {
124    ScoreableCheckpoint,
125    Finalization,
126}
127
128#[derive(Debug, Clone)]
129pub struct RuntimeConfig {
130    pub default_provider: String,
131    pub default_model: String,
132    pub reasoning: Option<String>,
133    pub auto_compact_token_limit: Option<u32>,
134    pub file_backed_dynamic_context: bool,
135    pub hosted_web_search: HostedWebSearchConfig,
136    pub tool_search: ToolSearchConfig,
137    pub provider_tool_search: HashMap<String, ToolSearchConfigOverlay>,
138    pub model_tool_search: HashMap<String, ToolSearchConfigOverlay>,
139    pub model_edit_tools: HashMap<String, String>,
140    pub model_parallel_tool_calls: HashMap<String, bool>,
141    pub model_profiles: HashMap<String, ModelHarnessProfile>,
142    pub tool_allowlist: Vec<String>,
143    /// Seconds a host-executed external tool call may stay unresolved before it fails with a timeout error.
144    pub external_tool_timeout_seconds: u64,
145    pub command_shell: String,
146    pub workspace: Option<String>,
147    pub policy_mode: PolicyMode,
148    /// Whether agent-swarm mode is active (roadmap 104). When on, the runtime
149    /// injects the swarm reminder into each turn's developer instructions so any
150    /// client benefits, not just the TUI.
151    pub agent_swarm_mode: bool,
152    /// Whether ultra mode is active. When on, the runtime injects proactive
153    /// multi-agent delegation policy for any model (not only Codex Sol/Terra
154    /// Ultra effort). Selecting Ultra reasoning on Sol/Terra still enables
155    /// proactive multi-agent for that model without requiring this flag.
156    pub ultra_mode: bool,
157    pub runtime_profile: RuntimeProfile,
158    pub inference_router: RuntimeInferenceRouterConfig,
159    pub speed_policy: RuntimeSpeedPolicyConfig,
160    pub dynamic_workflows: RuntimeDynamicWorkflowConfig,
161    pub reliability: RuntimeReliabilityConfig,
162    /// Positive wall-clock limit for a turn in any runtime profile. `None` or zero is unbounded.
163    pub turn_deadline_seconds: Option<u64>,
164    pub remote_runner_destination: Option<RunnerDestination>,
165    pub team_data_dir: Option<PathBuf>,
166    pub roadmap_data_dir: Option<PathBuf>,
167    pub media_generation: crate::media_generation::RuntimeMediaGenerationConfig,
168    /// `[review]` settings: the review sub-turn and its publishers.
169    pub review: crate::review::RuntimeReviewConfig,
170}
171
172impl Default for RuntimeConfig {
173    fn default() -> Self {
174        Self {
175            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
176            default_model: "mock".to_string(),
177            reasoning: None,
178            auto_compact_token_limit: None,
179            file_backed_dynamic_context: true,
180            hosted_web_search: HostedWebSearchConfig::cached(),
181            tool_search: ToolSearchConfig::default(),
182            provider_tool_search: HashMap::new(),
183            model_tool_search: HashMap::new(),
184            model_edit_tools: HashMap::new(),
185            model_parallel_tool_calls: HashMap::new(),
186            model_profiles: HashMap::new(),
187            tool_allowlist: Vec::new(),
188            external_tool_timeout_seconds: DEFAULT_EXTERNAL_TOOL_TIMEOUT_SECONDS,
189            command_shell: roder_api::command_shell::default_command_shell(),
190            workspace: None,
191            policy_mode: PolicyMode::Default,
192            agent_swarm_mode: false,
193            ultra_mode: false,
194            runtime_profile: RuntimeProfile::Interactive,
195            inference_router: RuntimeInferenceRouterConfig::default(),
196            speed_policy: RuntimeSpeedPolicyConfig::default(),
197            dynamic_workflows: RuntimeDynamicWorkflowConfig::default(),
198            reliability: RuntimeReliabilityConfig::default(),
199            turn_deadline_seconds: None,
200            remote_runner_destination: None,
201            team_data_dir: None,
202            roadmap_data_dir: None,
203            media_generation: crate::media_generation::RuntimeMediaGenerationConfig::default(),
204            review: crate::review::RuntimeReviewConfig::default(),
205        }
206    }
207}
208
209#[derive(Debug, Clone)]
210pub struct StartTurnRequest {
211    pub thread_id: ThreadId,
212    pub message: String,
213    pub images: Vec<InputImage>,
214    pub provider_override: Option<String>,
215    pub model_override: Option<String>,
216    pub reasoning_override: Option<String>,
217    pub workspace: String,
218    pub instructions: InstructionBundle,
219    /**
220     * Per-turn developer-authority context for this turn's InstructionBundle.
221     * Applies to every inference round of the turn, is never written to
222     * thread state, and does not carry over to later turns.
223     */
224    pub developer_context: Option<String>,
225    pub task_ledger_required: bool,
226    /**
227     * Per-turn provider service tier (OpenAI `service_tier`, e.g.
228     * `"priority"` for Fast mode). Carried to every inference round of the
229     * turn as `RuntimeHints::service_tier`; `None` keeps the provider
230     * default. Providers without a tier concept ignore it.
231     */
232    pub service_tier_override: Option<String>,
233}
234
235#[derive(Debug, Clone)]
236pub struct CreateThreadRequest {
237    pub title: Option<String>,
238    pub workspace: String,
239    pub workspace_id: Option<String>,
240    pub root_id: Option<String>,
241    pub provider: Option<String>,
242    pub model: Option<String>,
243    pub selection_mode: Option<ModelSelectionMode>,
244    /// Per-thread tool filter applied on top of the runtime allowlist. Empty = no filtering.
245    pub tool_allowlist: Vec<String>,
246    /// Host-supplied instructions added to the developer slot of every turn's inference request.
247    pub developer_instructions: Option<String>,
248    /// Host-executed tool specs advertised to the model on every turn of this thread.
249    pub external_tools: Vec<roder_api::tools::ToolSpec>,
250    /// Explicit remote-runner binding for the thread's native coding tools.
251    pub runner: Option<ThreadRunnerSelection>,
252}
253
254/**
255 * Thread-level remote-runner selection. The destination config is persisted
256 * with the thread, so secrets must reach the provider through its
257 * environment, not this config.
258 */
259#[derive(Debug, Clone)]
260pub struct ThreadRunnerSelection {
261    pub provider_id: String,
262    pub config: serde_json::Value,
263    /// Absolute path on the runner used as the thread's coding-tool workspace root.
264    pub workspace: String,
265    /**
266     * Extra absolute runner paths file reads may resolve under, beyond
267     * `workspace`. Writes and the working directory stay confined to
268     * `workspace`.
269     */
270    pub read_roots: Vec<String>,
271}
272
273#[derive(Debug, Clone, PartialEq, Eq)]
274pub struct PendingPlanExit {
275    pub thread_id: ThreadId,
276    pub turn_id: TurnId,
277    pub request_id: String,
278    pub target_mode: PolicyMode,
279    pub plan_summary: Option<String>,
280    pub next_steps: Vec<String>,
281    pub requested_at: OffsetDateTime,
282    pub expires_at: Option<OffsetDateTime>,
283}
284
285pub(crate) struct PendingToolApproval {
286    pub(crate) thread_id: ThreadId,
287    pub(crate) turn_id: TurnId,
288    pub(crate) tool_id: String,
289    pub(crate) tool_name: String,
290    pub(crate) call: roder_api::tools::ToolCall,
291    pub(crate) tx: oneshot::Sender<bool>,
292}
293
294pub(crate) struct PendingUserInput {
295    pub(crate) thread_id: ThreadId,
296    pub(crate) turn_id: TurnId,
297    pub(crate) tx: oneshot::Sender<serde_json::Value>,
298}
299
300/// Host answer to an external tool call delivered via `tools/resolve`.
301#[derive(Debug, Clone, PartialEq, Eq)]
302pub struct ExternalToolResolution {
303    pub output: String,
304    pub is_error: bool,
305}
306
307pub(crate) struct PendingExternalToolCall {
308    pub(crate) thread_id: ThreadId,
309    pub(crate) turn_id: TurnId,
310    pub(crate) tool_id: String,
311    pub(crate) tool_name: String,
312    pub(crate) tx: oneshot::Sender<ExternalToolResolution>,
313}
314
315#[derive(Clone)]
316struct ActiveTurnHandle {
317    thread_id: ThreadId,
318    abort: AbortHandle,
319    steers: Arc<Mutex<Vec<QueuedTurnSteer>>>,
320    drain: Arc<TurnDrainHandle>,
321}
322
323/// Tracks a task after it has been interrupted and removed from the interactive
324/// active-turn map. This lets users start a follow-up turn immediately while a
325/// bounded runtime shutdown can still await the original task's cleanup.
326struct TurnDrainHandle {
327    thread_id: ThreadId,
328    interrupt_requested: AtomicBool,
329    interrupt_reason: Mutex<Option<TurnLifecycleReason>>,
330    completed: AtomicBool,
331    completed_notify: Notify,
332}
333
334/// Result of a bounded runtime drain. `Clean` proves the runtime's own turn
335/// tasks have reached their cleanup paths; provider-specific child-process
336/// reaping remains an explicit provider capability.
337#[derive(Debug, Clone, PartialEq, Eq)]
338pub enum RuntimeDrainOutcome {
339    Clean {
340        interrupted_turn_ids: Vec<TurnId>,
341    },
342    DeadlineExceeded {
343        interrupted_turn_ids: Vec<TurnId>,
344        remaining_turn_ids: Vec<TurnId>,
345    },
346    /// At least one lifecycle transition initiated by this drain could not be
347    /// durably appended. The runtime still made its best cleanup attempt, but
348    /// callers must not claim a persisted terminal state as proof of recovery.
349    PersistenceFailed {
350        interrupted_turn_ids: Vec<TurnId>,
351        remaining_turn_ids: Vec<TurnId>,
352    },
353}
354
355#[derive(Clone)]
356struct QueuedTurnSteer {
357    message: UserMessage,
358    mailbox_ack: Option<MailboxDeliveryAck>,
359}
360
361#[derive(Clone)]
362struct MailboxDeliveryAck {
363    team_id: TeamId,
364    message_ids: Vec<String>,
365}
366
367#[derive(Clone)]
368struct InheritedTurnContext {
369    workspace: String,
370    instructions: InstructionBundle,
371    developer_context: Option<String>,
372}
373
374#[derive(Debug, Clone, Default, PartialEq, Eq)]
375pub struct ThreadActivity {
376    pub active_turn_id: Option<TurnId>,
377    pub active_flags: Vec<String>,
378}
379
380/// Per-thread settings persisted at thread creation and applied to every turn.
381#[derive(Debug, Clone, Default)]
382pub(crate) struct ThreadTurnOverrides {
383    pub(crate) tool_allowlist: Vec<String>,
384    pub(crate) developer_instructions: Option<String>,
385    pub(crate) external_tools: Vec<roder_api::tools::ToolSpec>,
386}
387
388#[derive(Debug, Clone, Copy, PartialEq, Eq)]
389pub(crate) enum TurnRunOutcome {
390    Completed,
391    Stopped,
392}
393
394impl PendingPlanExit {
395    pub fn new(
396        thread_id: ThreadId,
397        turn_id: TurnId,
398        request_id: String,
399        target_mode: PolicyMode,
400        plan_summary: Option<String>,
401        next_steps: Vec<String>,
402    ) -> Self {
403        let requested_at = OffsetDateTime::now_utc();
404        Self {
405            thread_id,
406            turn_id,
407            request_id,
408            target_mode,
409            plan_summary,
410            next_steps,
411            requested_at,
412            expires_at: Some(requested_at + default_plan_exit_timeout()),
413        }
414    }
415
416    pub fn is_expired(&self, now: OffsetDateTime) -> bool {
417        self.expires_at.is_some_and(|expires_at| now >= expires_at)
418    }
419}
420
421pub fn default_plan_exit_timeout() -> Duration {
422    Duration::minutes(10)
423}
424
425pub const DEFAULT_EXTERNAL_TOOL_TIMEOUT_SECONDS: u64 = 300;
426
427fn format_mailbox_messages(team: &TeamState, messages: &[TeamMailboxMessage]) -> String {
428    messages
429        .iter()
430        .map(|message| {
431            let recipient = team
432                .members
433                .iter()
434                .find(|member| member.id == message.to_member_id)
435                .map(canonical_team_member_path)
436                .unwrap_or_else(|| format!("/root/{}", message.to_member_id));
437            let sender = message
438                .from_member_id
439                .as_deref()
440                .and_then(|id| team.members.iter().find(|member| member.id == id))
441                .map(canonical_team_member_path)
442                .unwrap_or_else(|| "/root".to_string());
443            let message_type = match message.kind {
444                TeamMailboxMessageKind::Message => "MESSAGE",
445                TeamMailboxMessageKind::NewTask => "NEW_TASK",
446                TeamMailboxMessageKind::FinalAnswer => "FINAL_ANSWER",
447            };
448            format!(
449                "Message Type: {message_type}\nTask name: {recipient}\nSender: {sender}\nPayload:\n{}",
450                message.text
451            )
452        })
453        .collect::<Vec<_>>()
454        .join("\n\n")
455}
456
457fn canonical_team_member_path(member: &TeamMemberDescriptor) -> String {
458    if let Some(agent_path) = member
459        .agent_path
460        .as_deref()
461        .filter(|path| *path == "/root" || path.starts_with("/root/"))
462    {
463        return agent_path.to_string();
464    }
465    if member.role == TeamMemberRole::Lead {
466        return "/root".to_string();
467    }
468    member
469        .task_name
470        .as_deref()
471        .filter(|name| !name.trim().is_empty())
472        .map(|name| format!("/root/{}", name.trim().trim_matches('/')))
473        .unwrap_or_else(|| format!("/root/{}", member.id))
474}
475
476fn is_codex_v2_team(team: &TeamState) -> bool {
477    team.members.iter().any(|member| {
478        member
479            .model
480            .as_deref()
481            .is_some_and(|model| model_supports_reasoning_effort(model, REASONING_ULTRA))
482    })
483}
484
485fn runtime_local_team_view(
486    mut team: TeamState,
487    active_thread_ids: &std::collections::HashSet<ThreadId>,
488) -> TeamState {
489    for member in &mut team.members {
490        if member.status == TeamMemberStatus::Running
491            && !active_thread_ids.contains(&member.thread_id)
492        {
493            member.status = TeamMemberStatus::Interrupted;
494            member.current_turn_id = None;
495        }
496    }
497    team
498}
499
500type RunnerToolExecutionKey = (String, String);
501type RunnerToolExecutionMutex = Mutex<()>;
502type RunnerToolExecutionLockRegistry =
503    Mutex<HashMap<RunnerToolExecutionKey, Weak<RunnerToolExecutionMutex>>>;
504
505pub struct Runtime {
506    pub bus: EventBus,
507    pub registry: ExtensionRegistry,
508    config: RwLock<RuntimeConfig>,
509    pending_plan_exit: RwLock<Option<PendingPlanExit>>,
510    pub(crate) pending_tool_approvals: Mutex<HashMap<String, PendingToolApproval>>,
511    pub(crate) pending_user_inputs: Mutex<HashMap<String, PendingUserInput>>,
512    pub(crate) pending_external_tool_calls: Mutex<HashMap<String, PendingExternalToolCall>>,
513    /// Serializes turn admission with shutdown quiescing. A drain takes this
514    /// gate before flipping `accepting_turns` and snapshotting active work so a
515    /// turn that observed the old flag cannot be inserted after the snapshot.
516    turn_admission: Mutex<()>,
517    active_turns: RwLock<HashMap<TurnId, ActiveTurnHandle>>,
518    turn_drains: RwLock<HashMap<TurnId, Arc<TurnDrainHandle>>>,
519    /// Provider cleanup handles register synchronously through the inference
520    /// tool-executor context. They are intentionally separate from active turn
521    /// admission so an interrupted turn can remain drainable after a follow-up
522    /// turn starts on the same thread.
523    provider_turn_cleanups: std::sync::Mutex<HashMap<TurnId, Arc<dyn ProviderTurnCleanup>>>,
524    accepting_turns: AtomicBool,
525    /// Monotonic counter used by bounded drains to surface lifecycle-state
526    /// persistence failures without making an individual provider turn fail.
527    lifecycle_persistence_failures: AtomicUsize,
528    lifecycle_shutdown_drains: AtomicUsize,
529    lifecycle_clean_shutdowns: AtomicUsize,
530    lifecycle_deadline_exceeded: AtomicUsize,
531    lifecycle_persistence_failed_drains: AtomicUsize,
532    lifecycle_restart_reconciliations: AtomicUsize,
533    lifecycle_shutdown_drain_duration_ms_total: AtomicUsize,
534    provider_cleanup_confirmed: AtomicUsize,
535    provider_cleanup_timed_out: AtomicUsize,
536    provider_cleanup_unknown: AtomicUsize,
537    active_turns_changed: Notify,
538    active_turn_selections: RwLock<HashMap<TurnId, ModelSelectionMode>>,
539    /// Threads that have already dispatched a `SessionStart` local hook.
540    ///
541    /// `ThreadCreated` fires once, but `ThreadLoaded` fires on every read of the
542    /// thread from its store, so without this a session ran its setup hooks
543    /// several times — including after the turn had already stopped.
544    session_hook_started: RwLock<HashSet<ThreadId>>,
545    active_turn_contexts: RwLock<HashMap<TurnId, InheritedTurnContext>>,
546    /// Process-local execution boundary. Local/CLI runtimes default to true;
547    /// hosted runtime pools set this false so missing runner metadata fails
548    /// closed instead of exposing the host workspace.
549    allow_local_workspaces: AtomicBool,
550    // Spawn-time live authority retained for every reusable turn of a long-lived teammate.
551    // This stays process-local because developer_context is explicitly volatile and must never
552    // be persisted to thread state.
553    team_member_turn_contexts: Mutex<HashMap<ThreadId, InheritedTurnContext>>,
554    workspace: PathBuf,
555    pub(crate) teams: TeamManager,
556    agent_team_spawn_lock: Mutex<()>,
557    pub(crate) roadmaps: Mutex<roder_roadmap::RoadmapRuntime>,
558    /// Completed reviews, most recent last, so a later publish can resolve a
559    /// review id back to its findings. Bounded; see `review::run`.
560    pub(crate) reviews: Mutex<Vec<crate::review::ReviewRecord>>,
561    pub(crate) goals: Arc<RuntimeGoalController>,
562    context_artifacts: roder_api::artifacts::ContextArtifactStore,
563    pub(crate) thread_store: Option<Arc<dyn ThreadStore>>,
564    thread_item_cache: Mutex<ThreadItemCache>,
565    pub(crate) tool_registry: ToolRegistry,
566    media_generation: Arc<crate::media_generation::MediaGenerationService>,
567    pub(crate) skills: RwLock<SkillRegistry>,
568    /// Lazily-started bounded dispatch of emitted events to registry
569    /// `EventSink`s (process extensions etc.); see `event_sink_dispatch`.
570    event_sink_dispatcher: tokio::sync::OnceCell<crate::event_sink_dispatch::EventSinkDispatcher>,
571    pub(crate) compaction_hysteresis: std::sync::Mutex<HashMap<ThreadId, u32>>,
572    /// Per-thread agent-swarm mode overrides (roadmap 104). A thread present in
573    /// this map uses its stored value; absent threads fall back to the
574    /// runtime-global `RuntimeConfig.agent_swarm_mode` default. This mirrors the
575    /// team per-member policy-mode override idiom so swarm mode is per-thread
576    /// like the other `thread/*` operations, without a separate runtime.
577    agent_swarm_modes: RwLock<HashMap<ThreadId, bool>>,
578    /// Per-thread ultra mode overrides. A thread present in this map uses its
579    /// stored value; absent threads fall back to the runtime-global
580    /// `RuntimeConfig.ultra_mode` default.
581    ultra_modes: RwLock<HashMap<ThreadId, bool>>,
582    /**
583     * Live remote-runner sessions keyed by thread. Each per-thread mutex is
584     * held across provider initialization, making the first workspace-tool
585     * access a singleflight while allowing unrelated threads to initialize in
586     * parallel. Failed initialization is shared with current waiters, then
587     * evicted so a later tool call can retry.
588     */
589    runner_sessions: Arc<Mutex<HashMap<ThreadId, Arc<RunnerSessionSlot>>>>,
590    /**
591     * Outer tool-call locks keyed by the actual remote session, not the
592     * per-thread destination id. Hosted runtimes are tenant-scoped, so this
593     * serializes threads that share one sandbox without coupling different
594     * tenants or independent runner sessions.
595     */
596    runner_tool_execution_locks: RunnerToolExecutionLockRegistry,
597}
598
599#[derive(Clone)]
600struct CachedRunnerSession {
601    destination: RunnerDestination,
602    session: Arc<dyn RemoteRunnerSession>,
603}
604
605struct RunnerSessionSlot {
606    provider_id: String,
607    destination_id: String,
608    state: Mutex<RunnerSessionSlotState>,
609    initialized: Notify,
610}
611
612enum RunnerSessionSlotState {
613    Empty,
614    Initializing,
615    Ready(CachedRunnerSession),
616    Failed(Arc<str>),
617}
618
619impl RunnerSessionSlot {
620    fn empty(destination: &RunnerDestination) -> Self {
621        Self {
622            provider_id: destination.provider_id.clone(),
623            destination_id: destination.id.clone(),
624            state: Mutex::new(RunnerSessionSlotState::Empty),
625            initialized: Notify::new(),
626        }
627    }
628
629    fn ready(session: CachedRunnerSession) -> Self {
630        Self {
631            provider_id: session.destination.provider_id.clone(),
632            destination_id: session.destination.id.clone(),
633            state: Mutex::new(RunnerSessionSlotState::Ready(session)),
634            initialized: Notify::new(),
635        }
636    }
637
638    fn matches(&self, destination: &RunnerDestination) -> bool {
639        self.provider_id == destination.provider_id && self.destination_id == destination.id
640    }
641}
642
643impl Runtime {
644    pub fn new(registry: ExtensionRegistry, config: RuntimeConfig) -> anyhow::Result<Self> {
645        if registry.inference_engines.is_empty() {
646            anyhow::bail!("at least one inference engine must be registered");
647        }
648        validate_runtime_config_reasoning(&config)?;
649        validate_runtime_inference_router_config(&registry, &config)?;
650
651        let bus = EventBus::new(EVENT_BUS_CAPACITY);
652        let thread_store = registry
653            .thread_stores
654            .first()
655            .map(|factory| factory.create());
656        let mut tool_registry = ToolRegistry::default();
657        for contributor in &registry.tools {
658            contributor
659                .contribute(&mut tool_registry)
660                .with_context(|| format!("tool contributor {} failed", contributor.id()))?;
661        }
662        crate::agent_control_tools::contribute_agent_control_tools(&mut tool_registry)?;
663
664        let media_generation = Arc::new(crate::media_generation::MediaGenerationService::new(
665            registry.media_generator_providers.clone(),
666            config.media_generation.clone(),
667        ));
668        tool_registry.replace(Arc::new(
669            crate::media_generation::MediaGenerateImageTool::new(media_generation.clone()),
670        ));
671
672        let team_data_dir = config.team_data_dir.clone();
673        let workspace = config
674            .workspace
675            .clone()
676            .map(PathBuf::from)
677            .unwrap_or(std::env::current_dir()?);
678        let roadmap_data_dir = config
679            .roadmap_data_dir
680            .clone()
681            .unwrap_or_else(|| workspace.join(".roder"));
682        let context_artifacts = thread_store
683            .as_ref()
684            .and_then(|store| store.context_artifact_store())
685            .or_else(|| {
686                thread_store
687                    .as_ref()
688                    .and_then(|store| store.local_thread_root())
689                    .map(FilesystemContextArtifactStore::shared_thread_scoped)
690            })
691            .unwrap_or_else(|| {
692                FilesystemContextArtifactStore::shared_legacy(default_context_artifact_dir())
693            });
694        let goals = Arc::new(RuntimeGoalController::new(
695            bus.clone(),
696            thread_store.clone(),
697        ));
698        let runtime = Self {
699            bus,
700            registry,
701            config: RwLock::new(config),
702            pending_plan_exit: RwLock::new(None),
703            pending_tool_approvals: Mutex::new(HashMap::new()),
704            pending_user_inputs: Mutex::new(HashMap::new()),
705            pending_external_tool_calls: Mutex::new(HashMap::new()),
706            turn_admission: Mutex::new(()),
707            active_turns: RwLock::new(HashMap::new()),
708            turn_drains: RwLock::new(HashMap::new()),
709            provider_turn_cleanups: std::sync::Mutex::new(HashMap::new()),
710            accepting_turns: AtomicBool::new(true),
711            lifecycle_persistence_failures: AtomicUsize::new(0),
712            lifecycle_shutdown_drains: AtomicUsize::new(0),
713            lifecycle_clean_shutdowns: AtomicUsize::new(0),
714            lifecycle_deadline_exceeded: AtomicUsize::new(0),
715            lifecycle_persistence_failed_drains: AtomicUsize::new(0),
716            lifecycle_restart_reconciliations: AtomicUsize::new(0),
717            lifecycle_shutdown_drain_duration_ms_total: AtomicUsize::new(0),
718            provider_cleanup_confirmed: AtomicUsize::new(0),
719            provider_cleanup_timed_out: AtomicUsize::new(0),
720            provider_cleanup_unknown: AtomicUsize::new(0),
721            active_turns_changed: Notify::new(),
722            active_turn_selections: RwLock::new(HashMap::new()),
723            session_hook_started: RwLock::new(HashSet::new()),
724            active_turn_contexts: RwLock::new(HashMap::new()),
725            allow_local_workspaces: AtomicBool::new(true),
726            team_member_turn_contexts: Mutex::new(HashMap::new()),
727            workspace: workspace.clone(),
728            teams: TeamManager::new(
729                team_data_dir.unwrap_or_else(crate::teams::default_team_data_dir),
730            ),
731            agent_team_spawn_lock: Mutex::new(()),
732            roadmaps: Mutex::new(roder_roadmap::RoadmapRuntime::new(
733                workspace,
734                roadmap_data_dir,
735            )),
736            reviews: Mutex::new(Vec::new()),
737            goals,
738            context_artifacts,
739            thread_store,
740            thread_item_cache: Mutex::new(ThreadItemCache::default()),
741            tool_registry,
742            media_generation,
743            skills: RwLock::new(SkillRegistry::load(SkillRegistryOptions::new(
744                PathBuf::new(),
745            ))),
746            event_sink_dispatcher: tokio::sync::OnceCell::new(),
747            compaction_hysteresis: crate::compaction_runtime::compaction_hysteresis_state(),
748            agent_swarm_modes: RwLock::new(HashMap::new()),
749            ultra_modes: RwLock::new(HashMap::new()),
750            runner_sessions: Arc::new(Mutex::new(HashMap::new())),
751            runner_tool_execution_locks: Mutex::new(HashMap::new()),
752        };
753        runtime.bus.emit(RoderEvent::RuntimeStarted(RuntimeStarted {
754            timestamp: OffsetDateTime::now_utc(),
755        }));
756        for manifest in &runtime.registry.manifests {
757            runtime
758                .bus
759                .emit(RoderEvent::ExtensionRegistered(ExtensionRegistered {
760                    extension_id: manifest.id.clone(),
761                    timestamp: OffsetDateTime::now_utc(),
762                }));
763        }
764        Ok(runtime)
765    }
766
767    pub fn from_engine(engine: Arc<dyn InferenceEngine>) -> anyhow::Result<Self> {
768        let mut builder = roder_api::extension::ExtensionRegistryBuilder::new();
769        builder.inference_engine(engine);
770        Self::new(builder.build()?, RuntimeConfig::default())
771    }
772
773    pub fn fake() -> anyhow::Result<Self> {
774        Self::from_engine(Arc::new(FakeInferenceEngine))
775    }
776
777    pub fn subscribe_events(&self) -> tokio::sync::broadcast::Receiver<EventEnvelope> {
778        self.bus.subscribe()
779    }
780
781    /// Returns process-local, redacted lifecycle health counters. The snapshot
782    /// intentionally has fixed fields and contains no provider, command,
783    /// process, thread, or turn identifiers.
784    pub fn lifecycle_metrics(&self) -> LifecycleMetricsSnapshot {
785        LifecycleMetricsSnapshot {
786            shutdown_drain_count: self.lifecycle_shutdown_drains.load(Ordering::Acquire) as u64,
787            clean_shutdown_count: self.lifecycle_clean_shutdowns.load(Ordering::Acquire) as u64,
788            deadline_exceeded_count: self.lifecycle_deadline_exceeded.load(Ordering::Acquire)
789                as u64,
790            persistence_failed_count: self
791                .lifecycle_persistence_failed_drains
792                .load(Ordering::Acquire) as u64,
793            restart_reconciliation_count: self
794                .lifecycle_restart_reconciliations
795                .load(Ordering::Acquire) as u64,
796            lifecycle_persistence_failure_count: self
797                .lifecycle_persistence_failures
798                .load(Ordering::Acquire) as u64,
799            shutdown_drain_duration_ms_total: self
800                .lifecycle_shutdown_drain_duration_ms_total
801                .load(Ordering::Acquire) as u64,
802            provider_cleanup_confirmed_count: self
803                .provider_cleanup_confirmed
804                .load(Ordering::Acquire) as u64,
805            provider_cleanup_timed_out_count: self
806                .provider_cleanup_timed_out
807                .load(Ordering::Acquire) as u64,
808            provider_cleanup_unknown_count: self.provider_cleanup_unknown.load(Ordering::Acquire)
809                as u64,
810        }
811    }
812
813    async fn persist_turn_lifecycle_record(&self, record: &TurnLifecycleRecord) -> bool {
814        let Some(store) = &self.thread_store else {
815            return true;
816        };
817        let state = match record.extension_state() {
818            Ok(state) => state,
819            Err(_) => {
820                self.lifecycle_persistence_failures
821                    .fetch_add(1, Ordering::AcqRel);
822                return false;
823            }
824        };
825        // Lifecycle diagnostics must not turn a provider result into a failed
826        // turn merely because a third-party store lacks extension-state support.
827        // JSONL/Postgres stores persist this append-only record atomically at
828        // their own storage boundary.
829        if store
830            .append_extension_state(&record.thread_id, &state)
831            .await
832            .is_err()
833        {
834            self.lifecycle_persistence_failures
835                .fetch_add(1, Ordering::AcqRel);
836            return false;
837        }
838        true
839    }
840
841    async fn record_turn_lifecycle(
842        &self,
843        thread_id: ThreadId,
844        turn_id: TurnId,
845        state: TurnLifecycleState,
846        cleanup: TurnCleanupState,
847        reason: Option<TurnLifecycleReason>,
848    ) -> TurnLifecycleRecord {
849        self.record_turn_lifecycle_with_ownership(
850            thread_id,
851            turn_id,
852            state,
853            cleanup,
854            reason,
855            TurnCleanupOwnership::RuntimeTaskOnly,
856        )
857        .await
858    }
859
860    async fn record_turn_lifecycle_with_ownership(
861        &self,
862        thread_id: ThreadId,
863        turn_id: TurnId,
864        state: TurnLifecycleState,
865        cleanup: TurnCleanupState,
866        reason: Option<TurnLifecycleReason>,
867        ownership: TurnCleanupOwnership,
868    ) -> TurnLifecycleRecord {
869        let record = TurnLifecycleRecord::new(
870            thread_id,
871            turn_id,
872            state,
873            cleanup,
874            reason,
875            OffsetDateTime::now_utc(),
876        )
877        .with_ownership(ownership);
878        let _ = self.persist_turn_lifecycle_record(&record).await;
879        self.emit(RoderEvent::TurnLifecycleUpdated(record.clone()))
880            .await;
881        record
882    }
883
884    pub(crate) fn register_provider_turn_cleanup(
885        &self,
886        turn_id: &TurnId,
887        cleanup: Arc<dyn ProviderTurnCleanup>,
888    ) {
889        if let Ok(mut cleanups) = self.provider_turn_cleanups.lock() {
890            cleanups.insert(turn_id.clone(), cleanup);
891        }
892    }
893
894    fn provider_cleanup_ownership(&self, turn_id: &TurnId) -> TurnCleanupOwnership {
895        self.provider_turn_cleanups
896            .lock()
897            .ok()
898            .and_then(|cleanups| cleanups.get(turn_id).cloned())
899            .map(|cleanup| cleanup.ownership())
900            .unwrap_or(TurnCleanupOwnership::RuntimeTaskOnly)
901    }
902
903    async fn await_provider_turn_cleanup(
904        &self,
905        turn_id: &TurnId,
906    ) -> (TurnCleanupState, TurnCleanupOwnership) {
907        let cleanup = self
908            .provider_turn_cleanups
909            .lock()
910            .ok()
911            .and_then(|mut cleanups| cleanups.remove(turn_id));
912        let Some(cleanup) = cleanup else {
913            return (
914                TurnCleanupState::Unknown,
915                TurnCleanupOwnership::RuntimeTaskOnly,
916            );
917        };
918        let ownership = cleanup.ownership();
919        match tokio::time::timeout(PROVIDER_CLEANUP_TIMEOUT, cleanup.wait_for_cleanup()).await {
920            Ok(Ok(())) => {
921                self.provider_cleanup_confirmed
922                    .fetch_add(1, Ordering::AcqRel);
923                (
924                    TurnCleanupState::Completed,
925                    TurnCleanupOwnership::ProviderCleanupConfirmed,
926                )
927            }
928            Ok(Err(_)) => {
929                self.provider_cleanup_unknown.fetch_add(1, Ordering::AcqRel);
930                (TurnCleanupState::Unknown, ownership)
931            }
932            Err(_) => {
933                self.provider_cleanup_timed_out
934                    .fetch_add(1, Ordering::AcqRel);
935                (TurnCleanupState::TimedOut, ownership)
936            }
937        }
938    }
939
940    fn record_lifecycle_drain_outcome(
941        &self,
942        started_at: tokio::time::Instant,
943        outcome: &RuntimeDrainOutcome,
944    ) {
945        self.lifecycle_shutdown_drains
946            .fetch_add(1, Ordering::AcqRel);
947        self.lifecycle_shutdown_drain_duration_ms_total.fetch_add(
948            started_at.elapsed().as_millis().min(usize::MAX as u128) as usize,
949            Ordering::AcqRel,
950        );
951        match outcome {
952            RuntimeDrainOutcome::Clean { .. } => {
953                self.lifecycle_clean_shutdowns
954                    .fetch_add(1, Ordering::AcqRel);
955            }
956            RuntimeDrainOutcome::DeadlineExceeded { .. } => {
957                self.lifecycle_deadline_exceeded
958                    .fetch_add(1, Ordering::AcqRel);
959            }
960            RuntimeDrainOutcome::PersistenceFailed { .. } => {
961                self.lifecycle_persistence_failed_drains
962                    .fetch_add(1, Ordering::AcqRel);
963            }
964        }
965    }
966
967    fn lifecycle_record_for_event(event: &RoderEvent) -> Option<TurnLifecycleRecord> {
968        match event {
969            RoderEvent::TurnStarted(event) => Some(TurnLifecycleRecord::new(
970                event.thread_id.clone(),
971                event.turn_id.clone(),
972                TurnLifecycleState::Running,
973                TurnCleanupState::NotRequested,
974                None,
975                event.timestamp,
976            )),
977            // `run_turn` persists completed turns after it has awaited a
978            // registered provider cleanup handle. Do not let the generic event
979            // projection overwrite that acknowledgement with an unproven state.
980            RoderEvent::TurnCompleted(_) => None,
981            // Some failure paths already have an event before the turn wrapper
982            // reaches its cleanup acknowledgement. Preserve a durable fallback
983            // reason here; the wrapper appends a later record with the more
984            // precise cleanup outcome when a provider registered one.
985            RoderEvent::TurnFailed(event) => Some(TurnLifecycleRecord::new(
986                event.thread_id.clone(),
987                event.turn_id.clone(),
988                TurnLifecycleState::Failed,
989                TurnCleanupState::Unknown,
990                Some(if event.error_kind.as_deref() == Some("deadline_timeout") {
991                    TurnLifecycleReason::DeadlineExceeded
992                } else {
993                    TurnLifecycleReason::ProviderFailure
994                }),
995                event.timestamp,
996            )),
997            RoderEvent::TurnDeadlineExceeded(_) | RoderEvent::TurnInterrupted(_) => None,
998            RoderEvent::TurnLifecycleUpdated(_) => None,
999            _ => None,
1000        }
1001    }
1002
1003    pub fn registry(&self) -> &ExtensionRegistry {
1004        &self.registry
1005    }
1006
1007    pub fn media_generation(&self) -> Arc<crate::media_generation::MediaGenerationService> {
1008        self.media_generation.clone()
1009    }
1010
1011    pub fn context_artifacts(&self) -> roder_api::artifacts::ContextArtifactStore {
1012        self.context_artifacts.clone()
1013    }
1014
1015    pub async fn execute_workflow_tool(
1016        &self,
1017        thread_id: ThreadId,
1018        tool_name: &str,
1019        arguments: serde_json::Value,
1020    ) -> anyhow::Result<ToolResult> {
1021        let Some(executor) = self.tool_registry.get(tool_name) else {
1022            anyhow::bail!("tool not found: {tool_name}");
1023        };
1024        let tool_call = ToolCall {
1025            id: format!("slash-{tool_name}"),
1026            name: tool_name.to_string(),
1027            raw_arguments: serde_json::to_string(&arguments)?,
1028            arguments,
1029            thread_id: thread_id.clone(),
1030            turn_id: "slash-command".to_string(),
1031        };
1032        let runtime_config = self.status().await;
1033        let ctx = self.tool_execution_context(
1034            thread_id,
1035            "slash-command".to_string(),
1036            runtime_config.policy_mode,
1037            runtime_config.workspace.as_deref(),
1038            Some(&runtime_config.command_shell),
1039        );
1040        executor.execute(ctx, tool_call).await
1041    }
1042
1043    pub(crate) fn tool_execution_context(
1044        &self,
1045        thread_id: ThreadId,
1046        turn_id: TurnId,
1047        mode: PolicyMode,
1048        workspace: Option<&str>,
1049        command_shell: Option<&str>,
1050    ) -> ToolExecutionContext {
1051        let mut ctx = ToolExecutionContext::new(thread_id, turn_id, mode)
1052            .with_command_shell(command_shell.unwrap_or_default())
1053            .with_process_runner(Arc::new(LocalProcessRunner))
1054            .with_context_artifacts(self.context_artifacts.backend())
1055            .with_goal_controller(self.goals.clone())
1056            .with_subagent_trace_sink(Arc::new(RuntimeSubagentTraceSink::new(
1057                self.bus.clone(),
1058                self.thread_store.clone(),
1059            )))
1060            .with_swarm_progress_sink(Arc::new(RuntimeAgentSwarmProgressSink::new(
1061                self.bus.clone(),
1062                self.thread_store.clone(),
1063            )));
1064        if let Some(workspace) = workspace {
1065            ctx = ctx.with_workspace_handle(Arc::new(ScopedFilesystem::new(workspace)));
1066        }
1067        ctx
1068    }
1069
1070    pub async fn status(&self) -> RuntimeConfig {
1071        self.config.read().await.clone()
1072    }
1073
1074    pub async fn set_skills(&self, skills: SkillRegistry) {
1075        *self.skills.write().await = skills;
1076    }
1077
1078    pub async fn skills_snapshot(&self) -> SkillRegistry {
1079        self.skills.read().await.clone()
1080    }
1081
1082    pub fn workspace(&self) -> PathBuf {
1083        self.workspace.clone()
1084    }
1085
1086    /// Controls whether native workspace tools may execute without an
1087    /// explicit remote-runner binding. Hosted runtimes disable this after
1088    /// construction; ordinary local runtimes retain the compatible default.
1089    pub fn set_allow_local_workspaces(&self, allowed: bool) {
1090        self.allow_local_workspaces
1091            .store(allowed, Ordering::Relaxed);
1092    }
1093
1094    pub fn allows_local_workspaces(&self) -> bool {
1095        self.allow_local_workspaces.load(Ordering::Relaxed)
1096    }
1097
1098    pub async fn set_remote_runner_destination(&self, destination: Option<RunnerDestination>) {
1099        let lifecycle = destination.as_ref().map(|destination| RunnerLifecycle {
1100            destination_id: destination.id.clone(),
1101            provider_id: destination.provider_id.clone(),
1102            state: "configured".to_string(),
1103            session_id: None,
1104            timestamp: OffsetDateTime::now_utc(),
1105        });
1106        self.config.write().await.remote_runner_destination = destination;
1107        if let Some(lifecycle) = lifecycle {
1108            self.emit(RoderEvent::RunnerLifecycle(lifecycle)).await;
1109        } else {
1110            self.emit(RoderEvent::RunnerLifecycle(RunnerLifecycle {
1111                destination_id: "local".to_string(),
1112                provider_id: "local".to_string(),
1113                state: "local_fallback".to_string(),
1114                session_id: None,
1115                timestamp: OffsetDateTime::now_utc(),
1116            }))
1117            .await;
1118        }
1119    }
1120
1121    pub async fn set_file_backed_dynamic_context(&self, enabled: bool) -> RuntimeConfig {
1122        let mut cfg = self.config.write().await;
1123        cfg.file_backed_dynamic_context = enabled;
1124        cfg.clone()
1125    }
1126
1127    pub async fn set_command_shell(&self, shell: String) -> RuntimeConfig {
1128        let mut cfg = self.config.write().await;
1129        cfg.command_shell = shell;
1130        cfg.clone()
1131    }
1132
1133    pub async fn pending_plan_exit(&self) -> Option<PendingPlanExit> {
1134        let mut pending = self.pending_plan_exit.write().await;
1135        let current = pending.clone()?;
1136        if !current.is_expired(OffsetDateTime::now_utc()) {
1137            return Some(current);
1138        }
1139        *pending = None;
1140        drop(pending);
1141        self.emit_plan_exit_resolved(&current, false, self.status().await.policy_mode)
1142            .await;
1143        None
1144    }
1145
1146    pub async fn set_policy_mode(
1147        &self,
1148        mode: PolicyMode,
1149        reason: Option<String>,
1150    ) -> anyhow::Result<RuntimeConfig> {
1151        let mut cfg = self.config.write().await;
1152        let previous_mode = cfg.policy_mode;
1153        cfg.policy_mode = mode;
1154        let next = cfg.clone();
1155        drop(cfg);
1156        self.emit(RoderEvent::PolicyModeChanged(PolicyModeChanged {
1157            thread_id: "runtime".to_string(),
1158            turn_id: None,
1159            previous_mode,
1160            new_mode: mode,
1161            reason,
1162            timestamp: OffsetDateTime::now_utc(),
1163        }))
1164        .await;
1165        self.auto_resolve_pending_tool_approvals_for_mode(mode)
1166            .await;
1167        Ok(next)
1168    }
1169
1170    /// Toggle agent-swarm mode for the runtime (roadmap 104). When enabled, the
1171    /// swarm reminder is injected into each turn's developer instructions so
1172    /// every app-server/SDK client gets it, not only the TUI.
1173    pub async fn set_agent_swarm_mode(
1174        &self,
1175        enabled: bool,
1176        trigger: roder_api::subagents::AgentSwarmModeTrigger,
1177    ) -> anyhow::Result<RuntimeConfig> {
1178        let mut cfg = self.config.write().await;
1179        cfg.agent_swarm_mode = enabled;
1180        let next = cfg.clone();
1181        drop(cfg);
1182        self.emit(RoderEvent::AgentSwarmModeChanged(
1183            roder_api::subagents::AgentSwarmModeChanged {
1184                thread_id: "runtime".to_string(),
1185                turn_id: None,
1186                enabled,
1187                trigger,
1188                timestamp: OffsetDateTime::now_utc(),
1189            },
1190        ))
1191        .await;
1192        Ok(next)
1193    }
1194
1195    /// Toggle agent-swarm mode for a single thread (roadmap 104). The override is
1196    /// stored per thread so toggling swarm mode on one thread does not leak the
1197    /// reminder into other threads sharing the runtime; absent threads fall back
1198    /// to the runtime-global default. The emitted `AgentSwarmModeChanged` event
1199    /// carries the real `thread_id`.
1200    pub async fn set_agent_swarm_mode_for_thread(
1201        &self,
1202        thread_id: &str,
1203        enabled: bool,
1204        trigger: roder_api::subagents::AgentSwarmModeTrigger,
1205    ) -> bool {
1206        {
1207            let mut modes = self.agent_swarm_modes.write().await;
1208            modes.insert(thread_id.to_string(), enabled);
1209        }
1210        self.emit(RoderEvent::AgentSwarmModeChanged(
1211            roder_api::subagents::AgentSwarmModeChanged {
1212                thread_id: thread_id.to_string(),
1213                turn_id: None,
1214                enabled,
1215                trigger,
1216                timestamp: OffsetDateTime::now_utc(),
1217            },
1218        ))
1219        .await;
1220        enabled
1221    }
1222
1223    /// Resolve whether agent-swarm mode is active for `thread_id`: the per-thread
1224    /// override when present, otherwise the runtime-global default. The turn loop
1225    /// uses this to decide whether to inject the swarm reminder.
1226    pub async fn effective_agent_swarm_mode_for_thread(&self, thread_id: &str) -> bool {
1227        if let Some(enabled) = self.agent_swarm_modes.read().await.get(thread_id).copied() {
1228            return enabled;
1229        }
1230        self.status().await.agent_swarm_mode
1231    }
1232
1233    /// Toggle ultra mode for the runtime. When enabled, proactive multi-agent
1234    /// policy is injected into each turn's developer instructions for any
1235    /// model, so every app-server/SDK client gets it (not only the TUI).
1236    pub async fn set_ultra_mode(
1237        &self,
1238        enabled: bool,
1239        trigger: roder_api::subagents::UltraModeTrigger,
1240    ) -> anyhow::Result<RuntimeConfig> {
1241        let mut cfg = self.config.write().await;
1242        cfg.ultra_mode = enabled;
1243        let next = cfg.clone();
1244        drop(cfg);
1245        self.emit(RoderEvent::UltraModeChanged(
1246            roder_api::subagents::UltraModeChanged {
1247                thread_id: "runtime".to_string(),
1248                turn_id: None,
1249                enabled,
1250                trigger,
1251                timestamp: OffsetDateTime::now_utc(),
1252            },
1253        ))
1254        .await;
1255        Ok(next)
1256    }
1257
1258    /// Toggle ultra mode for a single thread. The override is stored per thread
1259    /// so toggling on one thread does not leak proactive multi-agent policy into
1260    /// other threads sharing the runtime; absent threads fall back to the
1261    /// runtime-global default.
1262    pub async fn set_ultra_mode_for_thread(
1263        &self,
1264        thread_id: &str,
1265        enabled: bool,
1266        trigger: roder_api::subagents::UltraModeTrigger,
1267    ) -> bool {
1268        {
1269            let mut modes = self.ultra_modes.write().await;
1270            modes.insert(thread_id.to_string(), enabled);
1271        }
1272        self.emit(RoderEvent::UltraModeChanged(
1273            roder_api::subagents::UltraModeChanged {
1274                thread_id: thread_id.to_string(),
1275                turn_id: None,
1276                enabled,
1277                trigger,
1278                timestamp: OffsetDateTime::now_utc(),
1279            },
1280        ))
1281        .await;
1282        enabled
1283    }
1284
1285    /// Resolve whether ultra mode is active for `thread_id`: the per-thread
1286    /// override when present, otherwise the runtime-global default. The turn
1287    /// loop uses this (together with Sol/Terra Ultra effort) to decide whether
1288    /// to inject proactive multi-agent policy.
1289    pub async fn effective_ultra_mode_for_thread(&self, thread_id: &str) -> bool {
1290        if let Some(enabled) = self.ultra_modes.read().await.get(thread_id).copied() {
1291            return enabled;
1292        }
1293        self.status().await.ultra_mode
1294    }
1295
1296    pub async fn set_hosted_web_search(
1297        &self,
1298        mode: HostedWebSearchMode,
1299    ) -> anyhow::Result<RuntimeConfig> {
1300        let mut cfg = self.config.write().await;
1301        cfg.hosted_web_search = HostedWebSearchConfig { mode };
1302        Ok(cfg.clone())
1303    }
1304
1305    async fn auto_resolve_pending_tool_approvals_for_mode(&self, mode: PolicyMode) {
1306        let gate = DefaultPolicyGate::new();
1307        let mut pending = self.pending_tool_approvals.lock().await;
1308        let approval_ids = pending
1309            .iter()
1310            .filter_map(|(approval_id, approval)| {
1311                let ctx = ToolExecutionContext::new(
1312                    approval.thread_id.clone(),
1313                    approval.turn_id.clone(),
1314                    mode,
1315                );
1316                matches!(
1317                    gate.decide(&approval.call, mode, &ctx),
1318                    PolicyDecision::AutoApproved { .. }
1319                )
1320                .then_some(approval_id.clone())
1321            })
1322            .collect::<Vec<_>>();
1323        let approvals = approval_ids
1324            .into_iter()
1325            .filter_map(|approval_id| {
1326                pending
1327                    .remove(&approval_id)
1328                    .map(|approval| (approval_id, approval))
1329            })
1330            .collect::<Vec<_>>();
1331        drop(pending);
1332
1333        for (approval_id, approval) in approvals {
1334            let ctx = ToolExecutionContext::new(
1335                approval.thread_id.clone(),
1336                approval.turn_id.clone(),
1337                mode,
1338            );
1339            let decision = gate.decide(&approval.call, mode, &ctx);
1340            self.emit(RoderEvent::PolicyDecisionRecorded(PolicyDecisionRecorded {
1341                thread_id: approval.thread_id.clone(),
1342                turn_id: approval.turn_id.clone(),
1343                tool_id: approval.tool_id.clone(),
1344                tool_name: approval.tool_name.clone(),
1345                mode,
1346                decision,
1347                timestamp: OffsetDateTime::now_utc(),
1348            }))
1349            .await;
1350            if mode == PolicyMode::Bypass {
1351                self.emit(RoderEvent::PolicyBypassActive(PolicyBypassActive {
1352                    thread_id: approval.thread_id.clone(),
1353                    turn_id: approval.turn_id.clone(),
1354                    tool_id: approval.tool_id.clone(),
1355                    tool_name: approval.tool_name.clone(),
1356                    timestamp: OffsetDateTime::now_utc(),
1357                }))
1358                .await;
1359            }
1360            self.emit(RoderEvent::ApprovalResolved(ApprovalResolved {
1361                thread_id: approval.thread_id,
1362                turn_id: approval.turn_id,
1363                approval_id,
1364                tool_id: approval.tool_id,
1365                tool_name: approval.tool_name,
1366                approved: true,
1367                timestamp: OffsetDateTime::now_utc(),
1368            }))
1369            .await;
1370            let _ = approval.tx.send(true);
1371        }
1372    }
1373
1374    pub async fn record_pending_plan_exit(&self, pending: PendingPlanExit) {
1375        *self.pending_plan_exit.write().await = Some(pending.clone());
1376        self.emit(RoderEvent::PolicyExitPlanRequested(
1377            PolicyExitPlanRequested {
1378                thread_id: pending.thread_id,
1379                turn_id: pending.turn_id,
1380                request_id: pending.request_id,
1381                target_mode: pending.target_mode,
1382                plan_summary: pending.plan_summary,
1383                next_steps: pending.next_steps,
1384                timestamp: OffsetDateTime::now_utc(),
1385            },
1386        ))
1387        .await;
1388    }
1389
1390    pub async fn resolve_pending_plan_exit(
1391        &self,
1392        request_id: &str,
1393        approved: bool,
1394    ) -> anyhow::Result<Option<PendingPlanExit>> {
1395        let mut pending = self.pending_plan_exit.write().await;
1396        let Some(current) = pending.clone() else {
1397            return Ok(None);
1398        };
1399        if current.request_id != request_id {
1400            anyhow::bail!("pending plan exit request {request_id:?} was not found");
1401        }
1402        *pending = None;
1403        drop(pending);
1404
1405        let approved = approved && !current.is_expired(OffsetDateTime::now_utc());
1406        let resolved_mode = if approved {
1407            let mut cfg = self.config.write().await;
1408            let previous_mode = cfg.policy_mode;
1409            cfg.policy_mode = current.target_mode;
1410            drop(cfg);
1411            self.emit(RoderEvent::PolicyModeChanged(PolicyModeChanged {
1412                thread_id: current.thread_id.clone(),
1413                turn_id: Some(current.turn_id.clone()),
1414                previous_mode,
1415                new_mode: current.target_mode,
1416                reason: Some("approved plan exit".to_string()),
1417                timestamp: OffsetDateTime::now_utc(),
1418            }))
1419            .await;
1420            self.auto_resolve_pending_tool_approvals_for_mode(current.target_mode)
1421                .await;
1422            current.target_mode
1423        } else {
1424            self.status().await.policy_mode
1425        };
1426        self.emit_plan_exit_resolved(&current, approved, resolved_mode)
1427            .await;
1428        Ok(Some(current))
1429    }
1430
1431    pub async fn resolve_tool_approval(
1432        &self,
1433        approval_id: &str,
1434        approved: bool,
1435    ) -> anyhow::Result<bool> {
1436        let pending = self.pending_tool_approvals.lock().await.remove(approval_id);
1437        let Some(pending) = pending else {
1438            return Ok(false);
1439        };
1440        self.emit(RoderEvent::ApprovalResolved(ApprovalResolved {
1441            thread_id: pending.thread_id,
1442            turn_id: pending.turn_id,
1443            approval_id: approval_id.to_string(),
1444            tool_id: pending.tool_id,
1445            tool_name: pending.tool_name,
1446            approved,
1447            timestamp: OffsetDateTime::now_utc(),
1448        }))
1449        .await;
1450        let _ = pending.tx.send(approved);
1451        Ok(true)
1452    }
1453
1454    pub async fn request_app_server_tool_approval(
1455        &self,
1456        call: ToolCall,
1457        reason: Option<String>,
1458    ) -> anyhow::Result<bool> {
1459        let approval_id = call.id.clone();
1460        let (tx, rx) = oneshot::channel();
1461        self.pending_tool_approvals.lock().await.insert(
1462            approval_id.clone(),
1463            PendingToolApproval {
1464                thread_id: call.thread_id.clone(),
1465                turn_id: call.turn_id.clone(),
1466                tool_id: call.id.clone(),
1467                tool_name: call.name.clone(),
1468                call: call.clone(),
1469                tx,
1470            },
1471        );
1472        self.emit(RoderEvent::ApprovalRequested(ApprovalRequested {
1473            thread_id: call.thread_id.clone(),
1474            turn_id: call.turn_id.clone(),
1475            approval_id,
1476            tool_id: call.id.clone(),
1477            tool_name: call.name.clone(),
1478            reason,
1479            timestamp: OffsetDateTime::now_utc(),
1480        }))
1481        .await;
1482        Ok(rx.await.unwrap_or(false))
1483    }
1484
1485    /// Completes a pending host-executed tool call (`tools/resolve`). Returns false when the
1486    /// request id is unknown, already resolved, timed out, or cancelled by a turn interrupt.
1487    pub async fn resolve_external_tool_call(
1488        &self,
1489        request_id: &str,
1490        resolution: ExternalToolResolution,
1491    ) -> anyhow::Result<bool> {
1492        let pending = self
1493            .pending_external_tool_calls
1494            .lock()
1495            .await
1496            .remove(request_id);
1497        let Some(pending) = pending else {
1498            return Ok(false);
1499        };
1500        self.emit(RoderEvent::ExternalToolCallResolved(
1501            ExternalToolCallResolved {
1502                thread_id: pending.thread_id,
1503                turn_id: pending.turn_id,
1504                request_id: request_id.to_string(),
1505                tool_id: pending.tool_id,
1506                tool_name: pending.tool_name,
1507                outcome: ExternalToolCallOutcome::Resolved,
1508                is_error: resolution.is_error,
1509                timestamp: OffsetDateTime::now_utc(),
1510            },
1511        ))
1512        .await;
1513        let _ = pending.tx.send(resolution);
1514        Ok(true)
1515    }
1516
1517    /// Drops every pending external tool call for the turn and reports them as cancelled so an
1518    /// interrupt does not leak requests waiting on `tools/resolve`.
1519    async fn cancel_pending_external_tool_calls_for_turn(&self, turn_id: &TurnId) {
1520        let cancelled = {
1521            let mut pending = self.pending_external_tool_calls.lock().await;
1522            let request_ids = pending
1523                .iter()
1524                .filter(|(_, call)| &call.turn_id == turn_id)
1525                .map(|(request_id, _)| request_id.clone())
1526                .collect::<Vec<_>>();
1527            request_ids
1528                .into_iter()
1529                .filter_map(|request_id| pending.remove(&request_id).map(|call| (request_id, call)))
1530                .collect::<Vec<_>>()
1531        };
1532        for (request_id, call) in cancelled {
1533            self.emit(RoderEvent::ExternalToolCallResolved(
1534                ExternalToolCallResolved {
1535                    thread_id: call.thread_id,
1536                    turn_id: call.turn_id,
1537                    request_id,
1538                    tool_id: call.tool_id,
1539                    tool_name: call.tool_name,
1540                    outcome: ExternalToolCallOutcome::Cancelled,
1541                    is_error: true,
1542                    timestamp: OffsetDateTime::now_utc(),
1543                },
1544            ))
1545            .await;
1546        }
1547    }
1548
1549    pub async fn resolve_user_input(
1550        &self,
1551        request_id: &str,
1552        answers: serde_json::Value,
1553    ) -> anyhow::Result<bool> {
1554        let pending = self.pending_user_inputs.lock().await.remove(request_id);
1555        let Some(pending) = pending else {
1556            return Ok(false);
1557        };
1558        self.emit(RoderEvent::UserInputResolved(UserInputResolved {
1559            thread_id: pending.thread_id,
1560            turn_id: pending.turn_id,
1561            request_id: request_id.to_string(),
1562            answers: answers.clone(),
1563            timestamp: OffsetDateTime::now_utc(),
1564        }))
1565        .await;
1566        let _ = pending.tx.send(answers);
1567        Ok(true)
1568    }
1569
1570    async fn emit_plan_exit_resolved(
1571        &self,
1572        current: &PendingPlanExit,
1573        approved: bool,
1574        resolved_mode: PolicyMode,
1575    ) {
1576        self.emit(RoderEvent::PolicyExitPlanResolved(PolicyExitPlanResolved {
1577            thread_id: current.thread_id.clone(),
1578            turn_id: current.turn_id.clone(),
1579            request_id: current.request_id.clone(),
1580            approved,
1581            target_mode: current.target_mode,
1582            resolved_mode,
1583            timestamp: OffsetDateTime::now_utc(),
1584        }))
1585        .await;
1586    }
1587
1588    pub async fn select_provider(
1589        &self,
1590        provider: String,
1591        model: Option<String>,
1592        reasoning: Option<String>,
1593    ) -> anyhow::Result<RuntimeConfig> {
1594        let next = self
1595            .preview_provider_selection(provider, model, reasoning)
1596            .await?;
1597        let mut cfg = self.config.write().await;
1598        *cfg = next;
1599        Ok(cfg.clone())
1600    }
1601
1602    pub async fn preview_provider_selection(
1603        &self,
1604        provider: String,
1605        model: Option<String>,
1606        reasoning: Option<String>,
1607    ) -> anyhow::Result<RuntimeConfig> {
1608        self.engine_for(&provider)?;
1609        let mut cfg = self.config.read().await.clone();
1610        cfg.default_provider = provider;
1611        if let Some(model) = model {
1612            cfg.default_model = model;
1613        }
1614        if let Some(reasoning) = reasoning {
1615            if reasoning == REASONING_NONE
1616                && !model_supports_reasoning(&cfg.default_model, &reasoning)
1617            {
1618                return Ok(cfg.clone());
1619            }
1620            validate_reasoning_effort(&cfg.default_model, &reasoning)?;
1621            cfg.reasoning = Some(reasoning);
1622        }
1623        Ok(cfg)
1624    }
1625
1626    pub async fn effective_reasoning(&self) -> String {
1627        let cfg = self.config.read().await;
1628        effective_reasoning_for_model(&cfg, &cfg.default_model)
1629    }
1630
1631    pub fn effective_reasoning_for_config(cfg: &RuntimeConfig) -> String {
1632        effective_reasoning_for_model(cfg, &cfg.default_model)
1633    }
1634
1635    pub async fn set_dynamic_workflow_effort(
1636        &self,
1637        effort_profile: DynamicWorkflowEffortProfile,
1638    ) -> RuntimeConfig {
1639        let mut cfg = self.config.write().await;
1640        cfg.dynamic_workflows.effort_profile = effort_profile;
1641        cfg.clone()
1642    }
1643
1644    pub async fn dynamic_workflow_trigger_decision(
1645        &self,
1646        message: &str,
1647    ) -> WorkflowTriggerDecision {
1648        let cfg = self.config.read().await;
1649        classify_workflow_trigger(message, &cfg.dynamic_workflows)
1650    }
1651
1652    pub async fn create_thread(&self, title: Option<String>) -> anyhow::Result<ThreadMetadata> {
1653        self.create_thread_with(CreateThreadRequest {
1654            title,
1655            workspace: self.workspace.display().to_string(),
1656            workspace_id: None,
1657            root_id: None,
1658            provider: None,
1659            model: None,
1660            selection_mode: None,
1661            tool_allowlist: Vec::new(),
1662            developer_instructions: None,
1663            external_tools: Vec::new(),
1664            runner: None,
1665        })
1666        .await
1667    }
1668
1669    /**
1670     * Resolves an explicit thread-runner selection into a persisted binding.
1671     * Fails fast at thread creation when the provider is missing or rejects
1672     * the destination, instead of surfacing the error on the first tool call.
1673     */
1674    async fn resolve_thread_runner_binding(
1675        &self,
1676        thread_id: &str,
1677        selection: ThreadRunnerSelection,
1678    ) -> anyhow::Result<ThreadRunnerBinding> {
1679        let provider = self
1680            .registry
1681            .remote_runner_providers
1682            .iter()
1683            .find(|provider| provider.id() == selection.provider_id)
1684            .cloned()
1685            .ok_or_else(|| {
1686                anyhow::anyhow!(
1687                    "remote runner provider {:?} is not installed",
1688                    selection.provider_id
1689                )
1690            })?;
1691        let workspace = selection.workspace.trim();
1692        anyhow::ensure!(
1693            std::path::Path::new(workspace).is_absolute(),
1694            "runner workspace must be an absolute path on the runner: {workspace:?}"
1695        );
1696        let mut read_roots = Vec::with_capacity(selection.read_roots.len());
1697        for read_root in &selection.read_roots {
1698            let trimmed = read_root.trim();
1699            anyhow::ensure!(
1700                std::path::Path::new(trimmed).is_absolute(),
1701                "runner read root must be an absolute path on the runner: {trimmed:?}"
1702            );
1703            read_roots.push(PathBuf::from(trimmed));
1704        }
1705        let destination = RunnerDestination {
1706            id: format!("thread-{thread_id}"),
1707            provider_id: selection.provider_id,
1708            config: selection.config,
1709            default_manifest: roder_api::remote_runner::RunnerManifest::default(),
1710        };
1711        provider.validate_destination(&destination).await?;
1712        Ok(ThreadRunnerBinding {
1713            destination,
1714            workspace: PathBuf::from(workspace),
1715            read_roots,
1716        })
1717    }
1718
1719    /**
1720     * Build a per-thread binding from a runtime-level destination (selected via
1721     * the TUI runner picker or config `default_destination`) when the
1722     * destination's provider advertises a default workspace. Returns `None` when
1723     * there is no destination or the provider opts out (no default workspace),
1724     * preserving the legacy behavior where such threads keep local tools.
1725     */
1726    async fn synthesize_runtime_runner_binding(
1727        &self,
1728        thread_id: &str,
1729        destination: Option<RunnerDestination>,
1730    ) -> anyhow::Result<Option<ThreadRunnerBinding>> {
1731        let Some(destination) = destination else {
1732            return Ok(None);
1733        };
1734        let Some(provider) = self
1735            .registry
1736            .remote_runner_providers
1737            .iter()
1738            .find(|provider| provider.id() == destination.provider_id)
1739        else {
1740            return Ok(None);
1741        };
1742        // An explicit workspace in the destination config wins over the
1743        // provider default; absence of both means the provider opts out.
1744        let workspace = destination
1745            .config
1746            .get("working_dir")
1747            .and_then(serde_json::Value::as_str)
1748            .map(str::to_string)
1749            .or_else(|| provider.default_workspace());
1750        let Some(workspace) = workspace else {
1751            return Ok(None);
1752        };
1753        let selection = ThreadRunnerSelection {
1754            provider_id: destination.provider_id.clone(),
1755            config: destination.config.clone(),
1756            workspace,
1757            read_roots: Vec::new(),
1758        };
1759        Ok(Some(
1760            self.resolve_thread_runner_binding(thread_id, selection)
1761                .await?,
1762        ))
1763    }
1764
1765    /// Validates a runner selection without creating a thread; the placeholder destination id only appears in error messages.
1766    pub async fn validate_thread_runner_selection(
1767        &self,
1768        selection: ThreadRunnerSelection,
1769    ) -> anyhow::Result<()> {
1770        self.resolve_thread_runner_binding("validate", selection)
1771            .await
1772            .map(|_| ())
1773    }
1774
1775    pub async fn create_thread_with(
1776        &self,
1777        req: CreateThreadRequest,
1778    ) -> anyhow::Result<ThreadMetadata> {
1779        let cfg = self.config.read().await.clone();
1780        let now = OffsetDateTime::now_utc();
1781        let workspace = validate_thread_workspace(&req.workspace)?;
1782        let provider = req.provider.unwrap_or(cfg.default_provider);
1783        let model = req.model.unwrap_or(cfg.default_model);
1784        let selection_mode = req
1785            .selection_mode
1786            .unwrap_or_else(|| ModelSelectionMode::manual(provider.clone(), model.clone(), None));
1787        let thread_id = uuid::Uuid::new_v4().to_string();
1788        let runner_binding = match req.runner {
1789            Some(selection) => Some(
1790                self.resolve_thread_runner_binding(&thread_id, selection)
1791                    .await?,
1792            ),
1793            // No explicit per-thread selection: if a runtime-level destination
1794            // is active and its provider advertises a default workspace, bind
1795            // the new thread so its coding tools route into the runner. This is
1796            // what makes a TUI/config-selected runner (e.g. Blaxel) actually
1797            // execute tools in the sandbox instead of locally. Providers that
1798            // return no default workspace stay unbound (legacy local-tools).
1799            None => {
1800                self.synthesize_runtime_runner_binding(
1801                    &thread_id,
1802                    cfg.remote_runner_destination.clone(),
1803                )
1804                .await?
1805            }
1806        };
1807        let runner_destination = runner_binding
1808            .as_ref()
1809            .map(|binding| binding.destination.clone())
1810            .or_else(|| cfg.remote_runner_destination.clone());
1811        let metadata = ThreadMetadata {
1812            thread_id,
1813            title: req.title,
1814            workspace,
1815            workspace_id: req.workspace_id,
1816            root_id: req.root_id,
1817            provider: Some(provider),
1818            model: Some(model),
1819            selection_mode: Some(selection_mode),
1820            tool_allowlist: req.tool_allowlist,
1821            developer_instructions: req.developer_instructions,
1822            external_tools: req.external_tools,
1823            runner_destination,
1824            runner_state: None,
1825            runner_binding,
1826            created_at: now,
1827            updated_at: now,
1828            message_count: 0,
1829            usage: None,
1830            parent_thread_id: None,
1831            forked_from_turn_id: None,
1832            workspace_fork: None,
1833        };
1834
1835        let metadata = if let Some(store) = &self.thread_store {
1836            store.create_thread(metadata).await?
1837        } else {
1838            metadata
1839        };
1840        self.emit(RoderEvent::ThreadCreated(ThreadCreated {
1841            thread_id: metadata.thread_id.clone(),
1842            timestamp: OffsetDateTime::now_utc(),
1843        }))
1844        .await;
1845        Ok(metadata)
1846    }
1847
1848    pub async fn list_threads(&self) -> anyhow::Result<Vec<ThreadMetadata>> {
1849        if let Some(store) = &self.thread_store {
1850            return store.list_threads().await;
1851        }
1852        Ok(Vec::new())
1853    }
1854
1855    pub async fn list_threads_page(
1856        &self,
1857        options: roder_api::thread::ThreadListOptions,
1858    ) -> anyhow::Result<roder_api::thread::ThreadListPage> {
1859        if let Some(store) = &self.thread_store {
1860            return store.list_threads_page(options).await;
1861        }
1862        Ok(roder_api::thread::ThreadListPage::default())
1863    }
1864
1865    pub async fn load_thread_metadata(
1866        &self,
1867        thread_id: &str,
1868    ) -> anyhow::Result<Option<roder_api::thread::ThreadMetadata>> {
1869        if let Some(store) = &self.thread_store {
1870            return store.load_thread_metadata(&thread_id.to_string()).await;
1871        }
1872        Ok(None)
1873    }
1874
1875    pub async fn archive_thread(&self, thread_id: &str) -> anyhow::Result<bool> {
1876        let archived = if let Some(store) = &self.thread_store {
1877            store.archive_thread(&thread_id.to_string()).await?
1878        } else {
1879            false
1880        };
1881        if archived {
1882            let workspace = self.config.read().await.workspace.clone();
1883            crate::hooks::run_lifecycle(
1884                self,
1885                &thread_id.to_string(),
1886                &SESSION_HOOK_TURN_ID.to_string(),
1887                workspace.as_deref(),
1888                "SessionEnd",
1889                Some("clear"),
1890                serde_json::json!({"reason": "archive"}),
1891            )
1892            .await;
1893            self.thread_item_cache
1894                .lock()
1895                .await
1896                .remove_thread(&thread_id.to_string());
1897            self.evict_runner_session(&thread_id.to_string()).await;
1898        }
1899        Ok(archived)
1900    }
1901
1902    pub async fn start_team(&self, req: TeamStartRequest) -> anyhow::Result<TeamState> {
1903        let cfg = self.config.read().await.clone();
1904        let workspace = self.workspace.display().to_string();
1905        let lead_thread_id = match req.lead_thread_id {
1906            Some(thread_id) => thread_id,
1907            None => {
1908                self.create_thread_with(CreateThreadRequest {
1909                    title: Some("Team lead".to_string()),
1910                    workspace: workspace.clone(),
1911                    workspace_id: None,
1912                    root_id: None,
1913                    provider: None,
1914                    model: None,
1915                    selection_mode: None,
1916                    tool_allowlist: Vec::new(),
1917                    developer_instructions: None,
1918                    external_tools: Vec::new(),
1919                    runner: None,
1920                })
1921                .await?
1922                .thread_id
1923            }
1924        };
1925        let team_id = uuid::Uuid::new_v4().to_string();
1926        let active_lead_turn_id = self.active_turn_for_thread(&lead_thread_id).await;
1927        let lead_selection = match active_lead_turn_id.as_ref() {
1928            Some(turn_id) => self
1929                .active_turn_selections
1930                .read()
1931                .await
1932                .get(turn_id)
1933                .cloned(),
1934            None => self.selection_mode_for_thread(&lead_thread_id).await?,
1935        };
1936        let lead_concrete_selection = lead_selection
1937            .as_ref()
1938            .map(ModelSelectionMode::concrete_selection);
1939        let mut lead = crate::teams::lead_member(
1940            lead_thread_id.clone(),
1941            lead_concrete_selection
1942                .as_ref()
1943                .map(|selection| selection.provider.clone())
1944                .or_else(|| Some(cfg.default_provider.clone())),
1945            lead_concrete_selection
1946                .as_ref()
1947                .map(|selection| selection.model.clone())
1948                .or_else(|| Some(cfg.default_model.clone())),
1949            cfg.policy_mode,
1950        );
1951        if let Some(turn_id) = active_lead_turn_id {
1952            lead.current_turn_id = Some(turn_id);
1953            lead.status = TeamMemberStatus::Running;
1954        }
1955        let mut members = vec![lead];
1956
1957        for (index, member) in req.members.into_iter().enumerate() {
1958            let thread = self
1959                .create_thread_with(CreateThreadRequest {
1960                    title: Some(member.name.clone()),
1961                    workspace: workspace.clone(),
1962                    workspace_id: None,
1963                    root_id: None,
1964                    provider: member.model_provider.clone(),
1965                    model: member.model.clone(),
1966                    selection_mode: None,
1967                    tool_allowlist: Vec::new(),
1968                    developer_instructions: None,
1969                    external_tools: Vec::new(),
1970                    runner: None,
1971                })
1972                .await?;
1973            let member_id = format!("member-{}", index + 1);
1974            let descriptor = crate::teams::teammate_member(
1975                member_id.clone(),
1976                member.name,
1977                thread.thread_id.clone(),
1978                member.model_provider.or(thread.provider),
1979                member.model.or(thread.model),
1980                cfg.policy_mode,
1981            );
1982            members.push(descriptor);
1983        }
1984
1985        let now = OffsetDateTime::now_utc();
1986        let team = self
1987            .teams
1988            .insert(TeamState {
1989                id: team_id.clone(),
1990                lead_thread_id: lead_thread_id.clone(),
1991                display_mode: req.display_mode,
1992                members,
1993                mailbox: Vec::new(),
1994                tasks: Vec::new(),
1995                created_at: now,
1996                updated_at: now,
1997            })
1998            .await?;
1999        self.emit(RoderEvent::TeamStarted(TeamStarted {
2000            team_id: team_id.clone(),
2001            lead_thread_id,
2002            display_mode: team.display_mode,
2003            members: team.members.clone(),
2004            tasks: team.tasks.clone(),
2005            timestamp: OffsetDateTime::now_utc(),
2006        }))
2007        .await;
2008        for member in team
2009            .members
2010            .iter()
2011            .filter(|member| member.role != roder_api::teams::TeamMemberRole::Lead)
2012        {
2013            self.emit(RoderEvent::TeamMemberStarted(TeamMemberStarted {
2014                team_id: team_id.clone(),
2015                member: member.clone(),
2016                timestamp: OffsetDateTime::now_utc(),
2017            }))
2018            .await;
2019        }
2020        Ok(team)
2021    }
2022
2023    pub async fn list_teams(&self) -> Vec<TeamState> {
2024        let active_thread_ids = self
2025            .active_turns
2026            .read()
2027            .await
2028            .values()
2029            .map(|handle| handle.thread_id.clone())
2030            .collect::<std::collections::HashSet<_>>();
2031        self.teams
2032            .list()
2033            .await
2034            .into_iter()
2035            .map(|team| runtime_local_team_view(team, &active_thread_ids))
2036            .collect()
2037    }
2038
2039    pub async fn read_team(&self, team_id: &str) -> Option<TeamState> {
2040        let active_thread_ids = self
2041            .active_turns
2042            .read()
2043            .await
2044            .values()
2045            .map(|handle| handle.thread_id.clone())
2046            .collect::<std::collections::HashSet<_>>();
2047        self.teams
2048            .get(team_id)
2049            .await
2050            .map(|team| runtime_local_team_view(team, &active_thread_ids))
2051    }
2052
2053    pub async fn start_team_member(
2054        &self,
2055        team_id: &str,
2056        req: TeamMemberStartRequest,
2057    ) -> anyhow::Result<TeamState> {
2058        self.start_team_member_with_selection(team_id, req, None)
2059            .await
2060    }
2061
2062    pub async fn message_team_member(
2063        self: &Arc<Self>,
2064        team_id: &str,
2065        member_id: &str,
2066        message: String,
2067    ) -> anyhow::Result<TurnId> {
2068        let team = self
2069            .read_team(team_id)
2070            .await
2071            .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2072        self.followup_team_member(&team.lead_thread_id, team_id, member_id, message)
2073            .await
2074    }
2075
2076    /// Queue a mailbox message without starting an idle agent. If the target is
2077    /// already running, the message is delivered at the next inference boundary.
2078    pub(crate) async fn queue_team_member_message(
2079        self: &Arc<Self>,
2080        caller_thread_id: &ThreadId,
2081        team_id: &str,
2082        member_id: &str,
2083        message: String,
2084    ) -> anyhow::Result<Option<TurnId>> {
2085        let _delivery_guard = self.agent_team_spawn_lock.lock().await;
2086        let team = self
2087            .read_team(team_id)
2088            .await
2089            .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2090        let member = team
2091            .members
2092            .iter()
2093            .find(|member| member.id == member_id)
2094            .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2095            .clone();
2096        if member.status == TeamMemberStatus::Closed {
2097            anyhow::bail!("subagent {} is closed", member.name);
2098        }
2099        let from_member_id = team
2100            .members
2101            .iter()
2102            .find(|candidate| candidate.thread_id == *caller_thread_id)
2103            .map(|candidate| candidate.id.clone());
2104        self.teams
2105            .append_mailbox_message(
2106                team_id,
2107                from_member_id,
2108                member_id.to_string(),
2109                TeamMailboxMessageKind::Message,
2110                message,
2111            )
2112            .await?;
2113        let Some(turn_id) = self.active_turn_for_thread(&member.thread_id).await else {
2114            return Ok(None);
2115        };
2116        self.deliver_pending_team_mailbox(team_id, member_id, member.thread_id, turn_id.clone())
2117            .await?;
2118        Ok(Some(turn_id))
2119    }
2120
2121    /// Deliver queued messages and start an idle agent, or steer its active turn.
2122    pub(crate) async fn followup_team_member(
2123        self: &Arc<Self>,
2124        caller_thread_id: &ThreadId,
2125        team_id: &str,
2126        member_id: &str,
2127        message: String,
2128    ) -> anyhow::Result<TurnId> {
2129        let _spawn_guard = self.agent_team_spawn_lock.lock().await;
2130        self.followup_team_member_locked(caller_thread_id, team_id, member_id, message)
2131            .await
2132    }
2133
2134    async fn followup_team_member_locked(
2135        self: &Arc<Self>,
2136        caller_thread_id: &ThreadId,
2137        team_id: &str,
2138        member_id: &str,
2139        message: String,
2140    ) -> anyhow::Result<TurnId> {
2141        let team = self
2142            .read_team(team_id)
2143            .await
2144            .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2145        let member = team
2146            .members
2147            .iter()
2148            .find(|member| member.id == member_id)
2149            .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2150            .clone();
2151        if member.status == TeamMemberStatus::Closed {
2152            anyhow::bail!("subagent {} is closed", member.name);
2153        }
2154        let from_member_id = team
2155            .members
2156            .iter()
2157            .find(|candidate| candidate.thread_id == *caller_thread_id)
2158            .map(|candidate| candidate.id.clone());
2159        self.teams
2160            .append_mailbox_message(
2161                team_id,
2162                from_member_id,
2163                member_id.to_string(),
2164                TeamMailboxMessageKind::NewTask,
2165                message,
2166            )
2167            .await?;
2168        if let Some(turn_id) = self.active_turn_for_thread(&member.thread_id).await {
2169            self.deliver_pending_team_mailbox(
2170                team_id,
2171                member_id,
2172                member.thread_id,
2173                turn_id.clone(),
2174            )
2175            .await?;
2176            return Ok(turn_id);
2177        }
2178
2179        self.ensure_codex_v2_team_capacity(&team, &member.id, None)
2180            .await?;
2181        let metadata = self.load_thread_metadata(&member.thread_id).await?;
2182        let inherited_context = self
2183            .team_member_turn_contexts
2184            .lock()
2185            .await
2186            .get(&member.thread_id)
2187            .cloned();
2188        let workspace = inherited_context
2189            .as_ref()
2190            .map(|context| context.workspace.clone())
2191            .or_else(|| metadata.as_ref().map(|metadata| metadata.workspace.clone()))
2192            .unwrap_or_else(|| self.workspace.display().to_string());
2193        let member_thread_id = member.thread_id;
2194        let turn_id = self
2195            .start_turn(StartTurnRequest {
2196                thread_id: member_thread_id.clone(),
2197                message: String::new(),
2198                images: Vec::new(),
2199                provider_override: member.model_provider,
2200                model_override: member.model,
2201                reasoning_override: metadata
2202                    .as_ref()
2203                    .and_then(|metadata| metadata.selection_mode.as_ref())
2204                    .and_then(ModelSelectionMode::reasoning)
2205                    .map(str::to_string),
2206                workspace,
2207                instructions: inherited_context
2208                    .as_ref()
2209                    .map(|context| context.instructions.clone())
2210                    .unwrap_or_else(crate::default_instructions),
2211                developer_context: inherited_context
2212                    .as_ref()
2213                    .and_then(|context| context.developer_context.clone()),
2214                task_ledger_required: false,
2215                service_tier_override: None,
2216            })
2217            .await?;
2218        Ok(turn_id)
2219    }
2220
2221    pub async fn set_team_member_policy_mode(
2222        &self,
2223        team_id: &str,
2224        member_id: &str,
2225        policy_mode: PolicyMode,
2226    ) -> anyhow::Result<TeamState> {
2227        self.teams
2228            .set_member_policy_mode(team_id, member_id, policy_mode)
2229            .await
2230    }
2231
2232    pub async fn interrupt_team_member(
2233        &self,
2234        team_id: &str,
2235        member_id: &str,
2236    ) -> anyhow::Result<Option<TurnId>> {
2237        let _interrupt_guard = self.agent_team_spawn_lock.lock().await;
2238        let team = self
2239            .read_team(team_id)
2240            .await
2241            .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2242        let member = team
2243            .members
2244            .iter()
2245            .find(|member| member.id == member_id)
2246            .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2247            .clone();
2248        if member.status != TeamMemberStatus::Running {
2249            return Ok(None);
2250        }
2251        let Some(turn_id) = member.current_turn_id.clone() else {
2252            return Ok(None);
2253        };
2254        if self
2255            .active_turn_for_thread(&member.thread_id)
2256            .await
2257            .as_ref()
2258            != Some(&turn_id)
2259        {
2260            return Ok(None);
2261        }
2262        self.interrupt_turn(member.thread_id.clone(), turn_id.clone())
2263            .await?;
2264        // Make queued mailbox work immediately eligible for the replacement turn while the
2265        // lifecycle lock still prevents a follow-up from starting. Delivery acknowledgements
2266        // are turn-owned, so a late acknowledgement from the aborted turn cannot steal a
2267        // reservation acquired by its replacement.
2268        self.teams
2269            .release_mailbox_reservations_for_turn(&turn_id)
2270            .await;
2271        self.teams
2272            .update_member(team_id, member_id, |member| {
2273                member.status = TeamMemberStatus::Interrupted;
2274                member.current_turn_id = None;
2275                member.terminal_error = None;
2276            })
2277            .await?;
2278        self.emit(RoderEvent::TeamMemberCompleted(TeamMemberCompleted {
2279            team_id: team_id.to_string(),
2280            member_id: member_id.to_string(),
2281            member_thread_id: member.thread_id,
2282            turn_id: Some(turn_id.clone()),
2283            status: TeamMemberStatus::Interrupted,
2284            final_message: member.final_message,
2285            error: None,
2286            timestamp: OffsetDateTime::now_utc(),
2287        }))
2288        .await;
2289        Ok(Some(turn_id))
2290    }
2291
2292    pub async fn close_team_member(
2293        &self,
2294        team_id: &str,
2295        member_id: &str,
2296    ) -> anyhow::Result<roder_api::teams::TeamMemberDescriptor> {
2297        let team = self
2298            .read_team(team_id)
2299            .await
2300            .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2301        let member = team
2302            .members
2303            .iter()
2304            .find(|member| member.id == member_id)
2305            .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2306            .clone();
2307        if member.role == roder_api::teams::TeamMemberRole::Lead {
2308            anyhow::bail!("team lead cannot be closed as a subagent");
2309        }
2310        let interrupted_turn_id = if member.status == TeamMemberStatus::Running {
2311            if let Some(turn_id) = member.current_turn_id.clone() {
2312                self.interrupt_turn(member.thread_id.clone(), turn_id.clone())
2313                    .await?;
2314                Some(turn_id)
2315            } else {
2316                None
2317            }
2318        } else {
2319            member.current_turn_id.clone()
2320        };
2321        let updated = self
2322            .teams
2323            .update_member(team_id, member_id, |member| {
2324                member.status = TeamMemberStatus::Closed;
2325                member.current_turn_id = None;
2326            })
2327            .await?;
2328        let closed = updated
2329            .members
2330            .iter()
2331            .find(|member| member.id == member_id)
2332            .cloned()
2333            .ok_or_else(|| anyhow::anyhow!("closed team member disappeared"))?;
2334        self.team_member_turn_contexts
2335            .lock()
2336            .await
2337            .remove(&closed.thread_id);
2338        self.emit(RoderEvent::TeamMemberCompleted(TeamMemberCompleted {
2339            team_id: team_id.to_string(),
2340            member_id: closed.id.clone(),
2341            member_thread_id: closed.thread_id.clone(),
2342            turn_id: interrupted_turn_id,
2343            status: TeamMemberStatus::Closed,
2344            final_message: closed.final_message.clone(),
2345            error: closed.terminal_error.clone(),
2346            timestamp: OffsetDateTime::now_utc(),
2347        }))
2348        .await;
2349        Ok(closed)
2350    }
2351
2352    pub async fn cleanup_team(&self, team_id: &str, force: bool) -> anyhow::Result<bool> {
2353        let Some(team) = self.read_team(team_id).await else {
2354            return Ok(false);
2355        };
2356        if !force
2357            && team
2358                .members
2359                .iter()
2360                .any(|member| member.status == TeamMemberStatus::Running)
2361        {
2362            anyhow::bail!("team {team_id:?} has active teammates; use forced cleanup");
2363        }
2364        if force {
2365            for member in team.members.iter().filter(|member| {
2366                member.role != roder_api::teams::TeamMemberRole::Lead
2367                    && member.status == TeamMemberStatus::Running
2368            }) {
2369                if let Some(turn_id) = member
2370                    .current_turn_id
2371                    .clone()
2372                    .or(self.active_turn_for_thread(&member.thread_id).await)
2373                {
2374                    let _ = self.interrupt_turn(member.thread_id.clone(), turn_id).await;
2375                }
2376            }
2377        }
2378        let removed = self.teams.remove(team_id).await?.is_some();
2379        if removed {
2380            let member_thread_ids = team
2381                .members
2382                .iter()
2383                .map(|member| member.thread_id.clone())
2384                .collect::<std::collections::HashSet<_>>();
2385            self.team_member_turn_contexts
2386                .lock()
2387                .await
2388                .retain(|thread_id, _| !member_thread_ids.contains(thread_id));
2389            self.emit(RoderEvent::TeamCleanupCompleted(TeamCleanupCompleted {
2390                team_id: team_id.to_string(),
2391                forced: force,
2392                timestamp: OffsetDateTime::now_utc(),
2393            }))
2394            .await;
2395        }
2396        Ok(removed)
2397    }
2398
2399    pub async fn effective_policy_mode_for_thread(&self, thread_id: &str) -> PolicyMode {
2400        if let Some(mode) = self.teams.policy_mode_for_thread(thread_id).await {
2401            return mode;
2402        }
2403        self.status().await.policy_mode
2404    }
2405
2406    async fn complete_team_member_turn_with_result(
2407        &self,
2408        thread_id: &ThreadId,
2409        turn_id: &TurnId,
2410        status: TeamMemberStatus,
2411        final_message: Option<String>,
2412        terminal_error: Option<String>,
2413    ) -> anyhow::Result<()> {
2414        let _delivery_guard = self.agent_team_spawn_lock.lock().await;
2415        let Some((team_id, member)) = self
2416            .teams
2417            .complete_member_turn(
2418                thread_id,
2419                turn_id,
2420                status,
2421                final_message.clone(),
2422                terminal_error.clone(),
2423            )
2424            .await?
2425        else {
2426            return Ok(());
2427        };
2428        if let Some(parent_thread_id) = member.parent_thread_id.as_ref()
2429            && let Some(team) = self.read_team(&team_id).await
2430            && let Some(parent) = team
2431                .members
2432                .iter()
2433                .find(|candidate| candidate.thread_id == *parent_thread_id)
2434        {
2435            let identity = member
2436                .agent_path
2437                .as_deref()
2438                .or(member.task_name.as_deref())
2439                .unwrap_or(&member.name);
2440            let mut report = format!("Agent {identity} finished with status {status:?}.");
2441            if let Some(message) = final_message.as_deref()
2442                && !message.trim().is_empty()
2443            {
2444                report.push_str("\n\nFinal result:\n");
2445                report.push_str(message);
2446            }
2447            if let Some(error) = terminal_error.as_deref()
2448                && !error.trim().is_empty()
2449            {
2450                report.push_str("\n\nTerminal error:\n");
2451                report.push_str(error);
2452            }
2453            let mailbox_appended = self
2454                .teams
2455                .append_mailbox_message(
2456                    &team_id,
2457                    Some(member.id.clone()),
2458                    parent.id.clone(),
2459                    TeamMailboxMessageKind::FinalAnswer,
2460                    report,
2461                )
2462                .await
2463                .is_ok();
2464            if mailbox_appended
2465                && let Some(parent_turn_id) = self.active_turn_for_thread(parent_thread_id).await
2466            {
2467                // The parent may finish between the active-turn lookup and enqueue. Its durable
2468                // mailbox entry remains pending for the next turn, while terminal observers must
2469                // still receive TeamMemberCompleted for this child.
2470                let _ = self
2471                    .deliver_pending_team_mailbox(
2472                        &team_id,
2473                        &parent.id,
2474                        parent_thread_id.clone(),
2475                        parent_turn_id,
2476                    )
2477                    .await;
2478            }
2479        }
2480        self.emit(RoderEvent::TeamMemberCompleted(TeamMemberCompleted {
2481            team_id,
2482            member_id: member.id,
2483            member_thread_id: member.thread_id,
2484            turn_id: Some(turn_id.clone()),
2485            status,
2486            final_message,
2487            error: terminal_error,
2488            timestamp: OffsetDateTime::now_utc(),
2489        }))
2490        .await;
2491        Ok(())
2492    }
2493
2494    /// Stops accepting new turns, requests interruption for every active turn,
2495    /// and waits for their runtime tasks to reach terminal cleanup up to
2496    /// `timeout`. Repeated calls are safe and continue draining the same set.
2497    pub async fn drain_active_turns(&self, timeout: std::time::Duration) -> RuntimeDrainOutcome {
2498        let started_at = tokio::time::Instant::now();
2499        let persistence_failures_before =
2500            self.lifecycle_persistence_failures.load(Ordering::Acquire);
2501        let turn_admission = self.turn_admission.lock().await;
2502        self.accepting_turns.store(false, Ordering::Release);
2503        let active = self
2504            .active_turns
2505            .read()
2506            .await
2507            .iter()
2508            .map(|(turn_id, handle)| (turn_id.clone(), handle.thread_id.clone()))
2509            .collect::<Vec<_>>();
2510        let draining = self
2511            .turn_drains
2512            .read()
2513            .await
2514            .iter()
2515            .map(|(turn_id, drain)| (turn_id.clone(), drain.thread_id.clone()))
2516            .collect::<Vec<_>>();
2517        drop(turn_admission);
2518
2519        let mut interrupted_turns = std::collections::BTreeMap::new();
2520        for (turn_id, thread_id) in active.iter().chain(draining.iter()) {
2521            interrupted_turns.insert(turn_id.clone(), thread_id.clone());
2522        }
2523        let interrupted_turn_ids = interrupted_turns.keys().cloned().collect::<Vec<_>>();
2524        for (turn_id, thread_id) in active {
2525            let _ = self
2526                .interrupt_turn_with_reason(thread_id, turn_id, TurnLifecycleReason::Shutdown)
2527                .await;
2528        }
2529
2530        let wait_for_empty = async {
2531            loop {
2532                let notified = self.active_turns_changed.notified();
2533                if self.active_turns.read().await.is_empty()
2534                    && self.turn_drains.read().await.is_empty()
2535                {
2536                    return;
2537                }
2538                notified.await;
2539            }
2540        };
2541        let outcome = if tokio::time::timeout(timeout, wait_for_empty).await.is_ok() {
2542            if self.lifecycle_persistence_failures.load(Ordering::Acquire)
2543                > persistence_failures_before
2544            {
2545                RuntimeDrainOutcome::PersistenceFailed {
2546                    interrupted_turn_ids,
2547                    remaining_turn_ids: Vec::new(),
2548                }
2549            } else {
2550                RuntimeDrainOutcome::Clean {
2551                    interrupted_turn_ids,
2552                }
2553            }
2554        } else {
2555            let active_remaining = self
2556                .active_turns
2557                .read()
2558                .await
2559                .iter()
2560                .map(|(turn_id, handle)| (turn_id.clone(), handle.drain.clone()))
2561                .collect::<Vec<_>>();
2562            let draining_remaining = self
2563                .turn_drains
2564                .read()
2565                .await
2566                .iter()
2567                .map(|(turn_id, drain)| (turn_id.clone(), drain.clone()))
2568                .collect::<Vec<_>>();
2569            let remaining = active_remaining
2570                .into_iter()
2571                .chain(draining_remaining)
2572                .collect::<std::collections::BTreeMap<_, _>>();
2573            let remaining_turn_ids = remaining.keys().cloned().collect::<Vec<_>>();
2574            for turn_id in &remaining_turn_ids {
2575                let handle = remaining
2576                    .get(turn_id)
2577                    .expect("remaining turn ID must have a drain handle");
2578                self.record_turn_lifecycle(
2579                    handle.thread_id.clone(),
2580                    turn_id.clone(),
2581                    TurnLifecycleState::InterruptRequested,
2582                    TurnCleanupState::TimedOut,
2583                    Some(TurnLifecycleReason::Shutdown),
2584                )
2585                .await;
2586            }
2587            if self.lifecycle_persistence_failures.load(Ordering::Acquire)
2588                > persistence_failures_before
2589            {
2590                RuntimeDrainOutcome::PersistenceFailed {
2591                    interrupted_turn_ids,
2592                    remaining_turn_ids,
2593                }
2594            } else {
2595                RuntimeDrainOutcome::DeadlineExceeded {
2596                    interrupted_turn_ids,
2597                    remaining_turn_ids,
2598                }
2599            }
2600        };
2601        self.record_lifecycle_drain_outcome(started_at, &outcome);
2602        outcome
2603    }
2604
2605    /// Enables turn submission after a prior drain attempt. This is intended
2606    /// for embedders that keep a runtime alive after a bounded drain timeout.
2607    pub fn resume_accepting_turns(&self) {
2608        self.accepting_turns.store(true, Ordering::Release);
2609    }
2610
2611    pub async fn turn_lifecycle_snapshot(
2612        &self,
2613        thread_id: &ThreadId,
2614    ) -> anyhow::Result<TurnLifecycleSnapshot> {
2615        let Some(store) = &self.thread_store else {
2616            return Ok(TurnLifecycleSnapshot::default());
2617        };
2618        let extension_states = store.load_extension_states(thread_id).await?;
2619        Ok(turn_lifecycle_snapshot(&extension_states))
2620    }
2621
2622    pub async fn load_thread(
2623        &self,
2624        thread_id: &ThreadId,
2625    ) -> anyhow::Result<Option<ThreadSnapshot>> {
2626        let loaded = if let Some(store) = &self.thread_store {
2627            store.load_thread(thread_id).await?
2628        } else {
2629            None
2630        };
2631        if let Some(snapshot) = loaded.as_ref() {
2632            let live_turn_ids = self
2633                .active_turns
2634                .read()
2635                .await
2636                .keys()
2637                .cloned()
2638                .chain(self.turn_drains.read().await.keys().cloned())
2639                .collect::<std::collections::HashSet<_>>();
2640            let lifecycle = turn_lifecycle_snapshot(&snapshot.extension_states);
2641            for record in lifecycle.records.into_iter().filter(|record| {
2642                record.state.requires_recovery() && !live_turn_ids.contains(&record.turn_id)
2643            }) {
2644                self.lifecycle_restart_reconciliations
2645                    .fetch_add(1, Ordering::AcqRel);
2646                self.record_turn_lifecycle(
2647                    record.thread_id,
2648                    record.turn_id,
2649                    TurnLifecycleState::RecoveryNeeded,
2650                    TurnCleanupState::Unknown,
2651                    Some(TurnLifecycleReason::RuntimeRestart),
2652                )
2653                .await;
2654            }
2655            self.emit(RoderEvent::ThreadLoaded(ThreadLoaded {
2656                thread_id: thread_id.clone(),
2657                timestamp: OffsetDateTime::now_utc(),
2658            }))
2659            .await;
2660        }
2661        Ok(loaded)
2662    }
2663
2664    pub async fn workspace_for_thread(&self, thread_id: &ThreadId) -> anyhow::Result<String> {
2665        if let Some(store) = &self.thread_store {
2666            let snapshot = store
2667                .load_thread(thread_id)
2668                .await?
2669                .ok_or_else(|| anyhow::anyhow!("thread not found: {thread_id}"));
2670            match snapshot {
2671                Ok(snapshot) => {
2672                    if let Some(metadata) = snapshot.metadata {
2673                        // Fork-backed threads fail closed before any write
2674                        // when their workspace was removed out-of-band.
2675                        if let Some(fork) = &metadata.workspace_fork
2676                            && fork.status == roder_api::forks::ForkStatus::Active
2677                            && !std::path::Path::new(&metadata.workspace).is_dir()
2678                        {
2679                            anyhow::bail!(
2680                                "workspace fork {} is missing its workspace at {}; restore it or \
2681                                 remove the fork before running turns in this thread",
2682                                fork.id,
2683                                metadata.workspace
2684                            );
2685                        }
2686                        return Ok(metadata.workspace);
2687                    }
2688                    eprintln!(
2689                        "thread metadata missing while resolving workspace for {thread_id}; falling back to runtime workspace"
2690                    );
2691                }
2692                Err(err) => {
2693                    eprintln!(
2694                        "thread missing while resolving workspace for {thread_id}: {err}; falling back to runtime workspace"
2695                    );
2696                }
2697            }
2698        }
2699        Ok(self.workspace.display().to_string())
2700    }
2701
2702    pub(crate) async fn selection_mode_for_thread(
2703        &self,
2704        thread_id: &ThreadId,
2705    ) -> anyhow::Result<Option<ModelSelectionMode>> {
2706        let Some(store) = &self.thread_store else {
2707            return Ok(None);
2708        };
2709        Ok(store
2710            .load_thread(thread_id)
2711            .await?
2712            .and_then(|snapshot| snapshot.metadata)
2713            .and_then(|metadata| {
2714                metadata
2715                    .selection_mode
2716                    .or_else(|| match (metadata.provider, metadata.model) {
2717                        (Some(provider), Some(model)) => {
2718                            Some(ModelSelectionMode::manual(provider, model, None))
2719                        }
2720                        _ => None,
2721                    })
2722            }))
2723    }
2724
2725    /// Concrete provider/model for configured-role subagents (`task`,
2726    /// `agent_swarm`). Prefers the live parent turn selection so children stay
2727    /// on SuperGrok/Grok (or whatever the lead is using) instead of process
2728    /// startup defaults such as openai/gpt-5.5.
2729    pub(crate) async fn parent_model_selection_for_subagents(
2730        &self,
2731        thread_id: &ThreadId,
2732        turn_id: &TurnId,
2733    ) -> Option<roder_api::inference::ModelSelection> {
2734        if let Some(selection) = self
2735            .active_turn_selections
2736            .read()
2737            .await
2738            .get(turn_id)
2739            .cloned()
2740        {
2741            return Some(selection.concrete_selection());
2742        }
2743        if let Ok(Some(selection)) = self.selection_mode_for_thread(thread_id).await {
2744            return Some(selection.concrete_selection());
2745        }
2746        let cfg = self.status().await;
2747        if cfg.default_provider.trim().is_empty() || cfg.default_model.trim().is_empty() {
2748            return None;
2749        }
2750        Some(roder_api::inference::ModelSelection {
2751            provider: cfg.default_provider,
2752            model: cfg.default_model,
2753        })
2754    }
2755
2756    /// Per-thread tool allowlist, developer instructions, and external tools persisted at thread creation.
2757    pub(crate) async fn thread_turn_overrides(
2758        &self,
2759        thread_id: &ThreadId,
2760    ) -> anyhow::Result<ThreadTurnOverrides> {
2761        let Some(store) = &self.thread_store else {
2762            return Ok(ThreadTurnOverrides::default());
2763        };
2764        Ok(store
2765            .load_thread_metadata(thread_id)
2766            .await?
2767            .map(|metadata| ThreadTurnOverrides {
2768                tool_allowlist: metadata.tool_allowlist,
2769                developer_instructions: metadata.developer_instructions,
2770                external_tools: metadata.external_tools,
2771            })
2772            .unwrap_or_default())
2773    }
2774
2775    pub async fn set_thread_selection_mode(
2776        &self,
2777        thread_id: &ThreadId,
2778        selection_mode: ModelSelectionMode,
2779    ) -> anyhow::Result<()> {
2780        let Some(store) = &self.thread_store else {
2781            return Ok(());
2782        };
2783        let Some(snapshot) = store.load_thread(thread_id).await? else {
2784            anyhow::bail!("thread not found: {thread_id}");
2785        };
2786        let Some(mut metadata) = snapshot.metadata else {
2787            return Ok(());
2788        };
2789        let concrete = selection_mode.concrete_selection();
2790        metadata.provider = Some(concrete.provider);
2791        metadata.model = Some(concrete.model);
2792        metadata.selection_mode = Some(selection_mode);
2793        metadata.updated_at = OffsetDateTime::now_utc();
2794        store.update_thread_metadata(metadata).await?;
2795        Ok(())
2796    }
2797
2798    async fn runner_session_for_thread(
2799        &self,
2800        thread_id: &ThreadId,
2801    ) -> anyhow::Result<Option<(RunnerDestination, Arc<dyn RemoteRunnerSession>)>> {
2802        let metadata = if let Some(store) = &self.thread_store {
2803            store.load_thread_metadata(thread_id).await?
2804        } else {
2805            None
2806        };
2807        // An explicit per-thread binding wins over the runtime-level destination.
2808        let destination = metadata
2809            .as_ref()
2810            .and_then(|metadata| metadata.runner_binding.as_ref())
2811            .map(|binding| binding.destination.clone())
2812            .or(self.config.read().await.remote_runner_destination.clone());
2813        let Some(destination) = destination else {
2814            self.evict_runner_session(thread_id).await;
2815            return Ok(None);
2816        };
2817        let provider = self
2818            .registry
2819            .remote_runner_providers
2820            .iter()
2821            .find(|provider| provider.id() == destination.provider_id)
2822            .cloned()
2823            .ok_or_else(|| {
2824                anyhow::anyhow!(
2825                    "remote runner provider {:?} is not installed",
2826                    destination.provider_id
2827                )
2828            })?;
2829        let persisted_state = metadata.and_then(|metadata| metadata.runner_state);
2830        let slot = {
2831            let mut sessions = self.runner_sessions.lock().await;
2832            match sessions.get(thread_id) {
2833                Some(slot) if slot.matches(&destination) => slot.clone(),
2834                _ => {
2835                    let slot = Arc::new(RunnerSessionSlot::empty(&destination));
2836                    sessions.insert(thread_id.clone(), slot.clone());
2837                    slot
2838                }
2839            }
2840        };
2841
2842        loop {
2843            let initialized = slot.initialized.notified();
2844            let mut state = slot.state.lock().await;
2845            match &*state {
2846                RunnerSessionSlotState::Ready(cached) => {
2847                    return Ok(Some((cached.destination.clone(), cached.session.clone())));
2848                }
2849                RunnerSessionSlotState::Failed(error) => {
2850                    return Err(anyhow::anyhow!(error.to_string()));
2851                }
2852                RunnerSessionSlotState::Initializing => {
2853                    drop(state);
2854                    initialized.await;
2855                }
2856                RunnerSessionSlotState::Empty => {
2857                    *state = RunnerSessionSlotState::Initializing;
2858                    drop(state);
2859                    self.spawn_runner_session_initialization(
2860                        thread_id.clone(),
2861                        destination.clone(),
2862                        provider.clone(),
2863                        persisted_state.clone(),
2864                        slot.clone(),
2865                    );
2866                }
2867            }
2868        }
2869    }
2870
2871    fn spawn_runner_session_initialization(
2872        &self,
2873        thread_id: ThreadId,
2874        destination: RunnerDestination,
2875        provider: Arc<dyn RemoteRunnerProvider>,
2876        persisted_state: Option<RunnerSessionState>,
2877        slot: Arc<RunnerSessionSlot>,
2878    ) {
2879        let thread_store = self.thread_store.clone();
2880        let runner_sessions = self.runner_sessions.clone();
2881        // The task intentionally outlives the turn that first requested the
2882        // workspace. Interrupting that turn must not strand the slot in
2883        // `Initializing` and deadlock every later tool or lifecycle call.
2884        drop(tokio::spawn(async move {
2885            let initialized = async {
2886                let session = if let Some(state) = persisted_state
2887                    && state.provider_id == destination.provider_id
2888                    && state.destination_id == destination.id
2889                {
2890                    match provider.resume_session(state).await {
2891                        Ok(session) => session,
2892                        Err(_) => provider.create_session(destination.clone()).await?,
2893                    }
2894                } else {
2895                    provider.create_session(destination.clone()).await?
2896                };
2897                let resolved = (destination.clone(), session.clone());
2898                // Persist before releasing the singleflight or dispatching
2899                // the first tool. A later process can rejoin even if the turn
2900                // that requested initialization has already been interrupted.
2901                Self::persist_runner_state_in_store(
2902                    thread_store.as_ref(),
2903                    &thread_id,
2904                    Some(&resolved),
2905                )
2906                .await?;
2907                Ok::<_, anyhow::Error>(CachedRunnerSession {
2908                    destination,
2909                    session,
2910                })
2911            }
2912            .await;
2913
2914            match initialized {
2915                Ok(cached) => {
2916                    *slot.state.lock().await = RunnerSessionSlotState::Ready(cached);
2917                    slot.initialized.notify_waiters();
2918                }
2919                Err(error) => {
2920                    let message: Arc<str> = Arc::from(format!("{error:#}"));
2921                    *slot.state.lock().await = RunnerSessionSlotState::Failed(message);
2922                    slot.initialized.notify_waiters();
2923                    let mut sessions = runner_sessions.lock().await;
2924                    let is_current = sessions
2925                        .get(&thread_id)
2926                        .is_some_and(|current| Arc::ptr_eq(current, &slot));
2927                    if is_current {
2928                        sessions.remove(&thread_id);
2929                    }
2930                }
2931            }
2932        }));
2933    }
2934
2935    async fn evict_runner_session(&self, thread_id: &ThreadId) {
2936        self.runner_sessions.lock().await.remove(thread_id);
2937    }
2938
2939    async fn cache_runner_session(
2940        &self,
2941        thread_id: &ThreadId,
2942        destination: RunnerDestination,
2943        session: Arc<dyn RemoteRunnerSession>,
2944    ) {
2945        let cached = CachedRunnerSession {
2946            destination,
2947            session,
2948        };
2949        self.runner_sessions.lock().await.insert(
2950            thread_id.clone(),
2951            Arc::new(RunnerSessionSlot::ready(cached)),
2952        );
2953    }
2954
2955    /// Read a thread's explicit runner binding without creating or resuming a
2956    /// session. Tool routing uses this before local previews so runner-backed
2957    /// and fail-closed hosted calls never inspect the host workspace.
2958    pub(crate) async fn runner_binding_for_thread(
2959        &self,
2960        thread_id: &ThreadId,
2961    ) -> anyhow::Result<Option<ThreadRunnerBinding>> {
2962        let Some(store) = &self.thread_store else {
2963            return Ok(None);
2964        };
2965        Ok(store
2966            .load_thread_metadata(thread_id)
2967            .await?
2968            .and_then(|metadata| metadata.runner_binding))
2969    }
2970
2971    /// Resolve a previously-read binding into a live remote workspace. The
2972    /// session remains lazy: callers invoke this only after policy approval.
2973    pub(crate) async fn remote_workspace_for_binding(
2974        &self,
2975        thread_id: &ThreadId,
2976        binding: ThreadRunnerBinding,
2977    ) -> anyhow::Result<Arc<RemoteWorkspace>> {
2978        let session = self
2979            .runner_session_for_thread(thread_id)
2980            .await?
2981            .map(|(_, session)| session)
2982            .ok_or_else(|| {
2983                anyhow::anyhow!("runner-bound thread {thread_id} has no runner session")
2984            })?;
2985        Ok(Arc::new(RemoteWorkspace {
2986            session,
2987            root: binding.workspace,
2988            read_roots: binding.read_roots,
2989        }))
2990    }
2991
2992    pub(crate) async fn runner_tool_execution_lock(
2993        &self,
2994        session: &dyn RemoteRunnerSession,
2995    ) -> Arc<RunnerToolExecutionMutex> {
2996        let state = session.state();
2997        let key = (state.provider_id, state.session_id);
2998        let mut locks = self.runner_tool_execution_locks.lock().await;
2999        if let Some(lock) = locks.get(&key).and_then(Weak::upgrade) {
3000            return lock;
3001        }
3002
3003        locks.retain(|_, lock| lock.strong_count() > 0);
3004        let lock = Arc::new(Mutex::new(()));
3005        locks.insert(key, Arc::downgrade(&lock));
3006        lock
3007    }
3008
3009    async fn persist_runner_state(
3010        &self,
3011        thread_id: &ThreadId,
3012        runner: Option<&(RunnerDestination, Arc<dyn RemoteRunnerSession>)>,
3013    ) -> anyhow::Result<()> {
3014        Self::persist_runner_state_in_store(self.thread_store.as_ref(), thread_id, runner).await
3015    }
3016
3017    async fn persist_runner_state_in_store(
3018        store: Option<&Arc<dyn ThreadStore>>,
3019        thread_id: &ThreadId,
3020        runner: Option<&(RunnerDestination, Arc<dyn RemoteRunnerSession>)>,
3021    ) -> anyhow::Result<()> {
3022        let Some((destination, session)) = runner else {
3023            return Ok(());
3024        };
3025        let Some(store) = store else {
3026            return Ok(());
3027        };
3028        let Some(snapshot) = store.load_thread(thread_id).await? else {
3029            return Ok(());
3030        };
3031        let Some(mut metadata) = snapshot.metadata else {
3032            return Ok(());
3033        };
3034        metadata.runner_destination = Some(destination.clone());
3035        metadata.runner_state = Some(session.state());
3036        metadata.updated_at = OffsetDateTime::now_utc();
3037        store.update_thread_metadata(metadata).await?;
3038        Ok(())
3039    }
3040
3041    fn remote_runner_provider_by_id(
3042        &self,
3043        provider_id: &str,
3044    ) -> Option<Arc<dyn RemoteRunnerProvider>> {
3045        self.registry
3046            .remote_runner_providers
3047            .iter()
3048            .find(|provider| provider.id() == provider_id)
3049            .cloned()
3050    }
3051
3052    /// Resolve the live runner session for a thread along with its provider,
3053    /// failing clearly when the thread is not runner-bound.
3054    async fn thread_runner_session(
3055        &self,
3056        thread_id: &ThreadId,
3057    ) -> anyhow::Result<(
3058        RunnerDestination,
3059        Arc<dyn RemoteRunnerProvider>,
3060        Arc<dyn RemoteRunnerSession>,
3061    )> {
3062        let Some((destination, session)) = self.runner_session_for_thread(thread_id).await? else {
3063            anyhow::bail!("thread {thread_id} is not bound to a remote runner");
3064        };
3065        let provider = self
3066            .remote_runner_provider_by_id(&destination.provider_id)
3067            .ok_or_else(|| {
3068                anyhow::anyhow!(
3069                    "remote runner provider {:?} is not installed",
3070                    destination.provider_id
3071                )
3072            })?;
3073        Ok((destination, provider, session))
3074    }
3075
3076    /// Pause a runner-bound thread's session toward standby and persist the
3077    /// post-pause state. Errors if the provider is not pausable.
3078    pub async fn pause_thread_runner(
3079        &self,
3080        thread_id: &ThreadId,
3081    ) -> anyhow::Result<RunnerSessionState> {
3082        let (destination, provider, session) = self.thread_runner_session(thread_id).await?;
3083        anyhow::ensure!(
3084            provider.capabilities().pausable,
3085            "remote runner provider {:?} does not support pausing",
3086            destination.provider_id
3087        );
3088        let state = session.pause().await?;
3089        self.persist_runner_state(thread_id, Some(&(destination, session)))
3090            .await?;
3091        Ok(state)
3092    }
3093
3094    /// Resume (wake) a runner-bound thread's paused session and persist state.
3095    pub async fn resume_thread_runner(
3096        &self,
3097        thread_id: &ThreadId,
3098    ) -> anyhow::Result<RunnerSessionState> {
3099        let (destination, _provider, session) = self.thread_runner_session(thread_id).await?;
3100        let state = session.resume().await?;
3101        self.persist_runner_state(thread_id, Some(&(destination, session)))
3102            .await?;
3103        Ok(state)
3104    }
3105
3106    /// Detach a runner-bound thread's session: persist the durable, rejoinable
3107    /// state and leave the remote sandbox alive. Errors if not detachable.
3108    pub async fn detach_thread_runner(
3109        &self,
3110        thread_id: &ThreadId,
3111    ) -> anyhow::Result<RunnerSessionState> {
3112        let (destination, provider, session) = self.thread_runner_session(thread_id).await?;
3113        anyhow::ensure!(
3114            provider.capabilities().detachable,
3115            "remote runner provider {:?} does not support detaching",
3116            destination.provider_id
3117        );
3118        let state = session.detach().await?;
3119        // Persist the detached state explicitly so a later turn or process
3120        // rejoins the same sandbox instead of provisioning a new one.
3121        if let Some(store) = &self.thread_store
3122            && let Some(snapshot) = store.load_thread(thread_id).await?
3123            && let Some(mut metadata) = snapshot.metadata
3124        {
3125            metadata.runner_destination = Some(destination);
3126            metadata.runner_state = Some(state.clone());
3127            metadata.updated_at = OffsetDateTime::now_utc();
3128            store.update_thread_metadata(metadata).await?;
3129        }
3130        self.evict_runner_session(thread_id).await;
3131        Ok(state)
3132    }
3133
3134    /// Rejoin a previously created sandbox from a thread's persisted runner
3135    /// state without provisioning a new one. An optional `sandbox` overrides the
3136    /// persisted sandbox name (recovery by name). Persists refreshed state.
3137    pub async fn rejoin_thread_runner(
3138        &self,
3139        thread_id: &ThreadId,
3140        sandbox: Option<String>,
3141    ) -> anyhow::Result<RunnerSessionState> {
3142        let store = self
3143            .thread_store
3144            .as_ref()
3145            .ok_or_else(|| anyhow::anyhow!("thread store is required to rejoin a runner"))?;
3146        let metadata = store
3147            .load_thread_metadata(thread_id)
3148            .await?
3149            .ok_or_else(|| anyhow::anyhow!("thread {thread_id} has no metadata"))?;
3150        let destination = metadata
3151            .runner_binding
3152            .as_ref()
3153            .map(|binding| binding.destination.clone())
3154            .or_else(|| metadata.runner_destination.clone())
3155            .ok_or_else(|| anyhow::anyhow!("thread {thread_id} is not bound to a remote runner"))?;
3156        let mut state = metadata
3157            .runner_state
3158            .clone()
3159            .ok_or_else(|| anyhow::anyhow!("thread {thread_id} has no persisted runner state"))?;
3160        if let Some(sandbox) = sandbox
3161            && let Some(object) = state.metadata.as_object_mut()
3162        {
3163            object.insert("sandbox_name".to_string(), serde_json::Value::from(sandbox));
3164        }
3165        let provider = self
3166            .remote_runner_provider_by_id(&destination.provider_id)
3167            .ok_or_else(|| {
3168                anyhow::anyhow!(
3169                    "remote runner provider {:?} is not installed",
3170                    destination.provider_id
3171                )
3172            })?;
3173        let session = provider.rejoin_session(state).await?;
3174        self.persist_runner_state(thread_id, Some(&(destination.clone(), session.clone())))
3175            .await?;
3176        self.cache_runner_session(thread_id, destination, session.clone())
3177            .await;
3178        Ok(session.state())
3179    }
3180
3181    async fn record_thread_usage_metadata(
3182        &self,
3183        thread_id: &ThreadId,
3184        usage: &TokenUsage,
3185    ) -> anyhow::Result<()> {
3186        if usage.is_empty() {
3187            return Ok(());
3188        }
3189        let Some(store) = &self.thread_store else {
3190            return Ok(());
3191        };
3192        let Some(snapshot) = store.load_thread(thread_id).await? else {
3193            return Ok(());
3194        };
3195        let Some(mut metadata) = snapshot.metadata else {
3196            return Ok(());
3197        };
3198        metadata
3199            .usage
3200            .get_or_insert_with(ThreadUsageMetadata::default)
3201            .add_token_usage(usage);
3202        metadata.updated_at = OffsetDateTime::now_utc();
3203        store.update_thread_metadata(metadata).await?;
3204        Ok(())
3205    }
3206
3207    pub fn start_turn(
3208        self: &Arc<Self>,
3209        mut req: StartTurnRequest,
3210    ) -> BoxFuture<'_, anyhow::Result<TurnId>> {
3211        Box::pin(async move {
3212            let turn_admission = self.turn_admission.lock().await;
3213            anyhow::ensure!(
3214                self.accepting_turns.load(Ordering::Acquire),
3215                "runtime is quiescing and cannot accept new turns"
3216            );
3217            req.workspace = validate_thread_workspace(&req.workspace)?;
3218            let team_member = self.teams.member_for_thread(&req.thread_id).await;
3219            let cfg = self.config.read().await.clone();
3220            let provider = req
3221                .provider_override
3222                .clone()
3223                .unwrap_or_else(|| cfg.default_provider.clone());
3224            self.engine_for(&provider)?;
3225            let turn_id = uuid::Uuid::new_v4().to_string();
3226            let mut initial_mailbox_ack = None;
3227            if let Some((team_id, member)) = &team_member {
3228                let pending = self
3229                    .teams
3230                    .reserve_pending_mailbox_messages(team_id, &member.id, &turn_id)
3231                    .await?;
3232                if !pending.is_empty()
3233                    && let Some(team) = self.read_team(team_id).await
3234                {
3235                    let mailbox = format_mailbox_messages(&team, &pending);
3236                    req.message = if req.message.trim().is_empty() {
3237                        mailbox
3238                    } else {
3239                        format!("{mailbox}\n\n[Direct task input]\n{}", req.message)
3240                    };
3241                    initial_mailbox_ack = Some(MailboxDeliveryAck {
3242                        team_id: team_id.clone(),
3243                        message_ids: pending.iter().map(|message| message.id.clone()).collect(),
3244                    });
3245                }
3246            }
3247            let (abort_handle, abort_registration) = AbortHandle::new_pair();
3248            let drain = Arc::new(TurnDrainHandle {
3249                thread_id: req.thread_id.clone(),
3250                interrupt_requested: AtomicBool::new(false),
3251                interrupt_reason: Mutex::new(None),
3252                completed: AtomicBool::new(false),
3253                completed_notify: Notify::new(),
3254            });
3255            let active = ActiveTurnHandle {
3256                thread_id: req.thread_id.clone(),
3257                abort: abort_handle,
3258                steers: Arc::new(Mutex::new(Vec::new())),
3259                drain,
3260            };
3261            self.active_turns
3262                .write()
3263                .await
3264                .insert(turn_id.clone(), active);
3265            self.record_turn_lifecycle(
3266                req.thread_id.clone(),
3267                turn_id.clone(),
3268                TurnLifecycleState::Running,
3269                TurnCleanupState::NotRequested,
3270                None,
3271            )
3272            .await;
3273            self.active_turn_contexts.write().await.insert(
3274                turn_id.clone(),
3275                InheritedTurnContext {
3276                    workspace: req.workspace.clone(),
3277                    instructions: req.instructions.clone(),
3278                    developer_context: req.developer_context.clone(),
3279                },
3280            );
3281            if let Some((team_id, member)) = team_member {
3282                let updated = match self
3283                    .teams
3284                    .update_member(&team_id, &member.id, |member| {
3285                        member.current_turn_id = Some(turn_id.clone());
3286                        member.status = TeamMemberStatus::Running;
3287                        member.final_message = None;
3288                        member.terminal_error = None;
3289                    })
3290                    .await
3291                {
3292                    Ok(updated) => updated,
3293                    Err(error) => {
3294                        self.active_turns.write().await.remove(&turn_id);
3295                        self.active_turn_contexts.write().await.remove(&turn_id);
3296                        self.teams
3297                            .release_mailbox_reservations_for_turn(&turn_id)
3298                            .await;
3299                        return Err(error);
3300                    }
3301                };
3302                if let Some(member) = updated
3303                    .members
3304                    .into_iter()
3305                    .find(|candidate| candidate.id == member.id)
3306                {
3307                    self.emit(RoderEvent::TeamMemberStatusChanged(
3308                        TeamMemberStatusChanged {
3309                            team_id,
3310                            member_id: member.id,
3311                            member_thread_id: member.thread_id,
3312                            status: TeamMemberStatus::Running,
3313                            timestamp: OffsetDateTime::now_utc(),
3314                        },
3315                    ))
3316                    .await;
3317                }
3318            }
3319            let runtime = Arc::clone(self);
3320            let turn_req = req;
3321            let thread_id_for_task = turn_req.thread_id.clone();
3322            let turn_id_for_task = turn_id.clone();
3323            tokio::spawn(async move {
3324                let result = Abortable::new(
3325                    runtime.run_turn(turn_req, turn_id_for_task.clone(), initial_mailbox_ack),
3326                    abort_registration,
3327                )
3328                .await;
3329                /*
3330                 * A failed sibling in a parallel tool batch drops in-flight external tool
3331                 * futures (`try_join_all` in `route_tool_calls`), stranding their
3332                 * `pending_external_tool_calls` entries. Sweep before reporting the turn
3333                 * outcome so every `thread/toolExecutionRequested` gets a terminal
3334                 * resolution; on clean completion the map holds nothing for this turn.
3335                 */
3336                runtime
3337                    .cancel_pending_external_tool_calls_for_turn(&turn_id_for_task)
3338                    .await;
3339                let completed = matches!(&result, Ok(Ok(TurnRunOutcome::Completed)));
3340                match &result {
3341                    Ok(Err(err)) => {
3342                        let (cleanup, ownership) =
3343                            runtime.await_provider_turn_cleanup(&turn_id_for_task).await;
3344                        // run_turn emits failures after the stream starts; this covers setup/startup errors.
3345                        runtime
3346                            .emit(RoderEvent::TurnFailed(TurnFailed {
3347                                thread_id: thread_id_for_task.clone(),
3348                                turn_id: turn_id_for_task.clone(),
3349                                error: err.to_string(),
3350                                error_kind: None,
3351                                usage: None,
3352                                timestamp: OffsetDateTime::now_utc(),
3353                            }))
3354                            .await;
3355                        runtime
3356                            .record_turn_lifecycle_with_ownership(
3357                                thread_id_for_task.clone(),
3358                                turn_id_for_task.clone(),
3359                                TurnLifecycleState::Failed,
3360                                cleanup,
3361                                Some(TurnLifecycleReason::ProviderFailure),
3362                                ownership,
3363                            )
3364                            .await;
3365                        let _ = runtime
3366                            .complete_team_member_turn_with_result(
3367                                &thread_id_for_task,
3368                                &turn_id_for_task,
3369                                TeamMemberStatus::Failed,
3370                                None,
3371                                Some(err.to_string()),
3372                            )
3373                            .await;
3374                    }
3375                    Ok(Ok(TurnRunOutcome::Stopped)) => {
3376                        let (cleanup, ownership) =
3377                            runtime.await_provider_turn_cleanup(&turn_id_for_task).await;
3378                        runtime
3379                            .record_turn_lifecycle_with_ownership(
3380                                thread_id_for_task.clone(),
3381                                turn_id_for_task.clone(),
3382                                TurnLifecycleState::Failed,
3383                                cleanup,
3384                                Some(TurnLifecycleReason::ProviderFailure),
3385                                ownership,
3386                            )
3387                            .await;
3388                        let _ = runtime
3389                            .complete_team_member_turn_with_result(
3390                                &thread_id_for_task,
3391                                &turn_id_for_task,
3392                                TeamMemberStatus::Failed,
3393                                None,
3394                                Some("turn stopped before completion".to_string()),
3395                            )
3396                            .await;
3397                    }
3398                    Err(_) => {
3399                        let reason = if let Some(handle) = runtime
3400                            .turn_drains
3401                            .read()
3402                            .await
3403                            .get(&turn_id_for_task)
3404                            .cloned()
3405                        {
3406                            handle
3407                                .interrupt_reason
3408                                .lock()
3409                                .await
3410                                .unwrap_or(TurnLifecycleReason::RuntimeFailure)
3411                        } else {
3412                            TurnLifecycleReason::RuntimeFailure
3413                        };
3414                        let (cleanup, ownership) =
3415                            runtime.await_provider_turn_cleanup(&turn_id_for_task).await;
3416                        runtime
3417                            .record_turn_lifecycle_with_ownership(
3418                                thread_id_for_task.clone(),
3419                                turn_id_for_task.clone(),
3420                                TurnLifecycleState::Interrupted,
3421                                cleanup,
3422                                Some(reason),
3423                                ownership,
3424                            )
3425                            .await;
3426                        runtime
3427                            .emit(RoderEvent::TurnInterrupted(TurnInterrupted {
3428                                thread_id: thread_id_for_task.clone(),
3429                                turn_id: turn_id_for_task.clone(),
3430                                timestamp: OffsetDateTime::now_utc(),
3431                            }))
3432                            .await;
3433                        let _ = runtime
3434                            .complete_team_member_turn_with_result(
3435                                &thread_id_for_task,
3436                                &turn_id_for_task,
3437                                TeamMemberStatus::Interrupted,
3438                                None,
3439                                None,
3440                            )
3441                            .await;
3442                    }
3443                    Ok(Ok(TurnRunOutcome::Completed)) => {}
3444                }
3445                runtime
3446                    .teams
3447                    .release_mailbox_reservations_for_turn(&turn_id_for_task)
3448                    .await;
3449                runtime.active_turns.write().await.remove(&turn_id_for_task);
3450                if let Some(drain) = runtime.turn_drains.write().await.remove(&turn_id_for_task) {
3451                    drain.completed.store(true, Ordering::Release);
3452                    drain.completed_notify.notify_waiters();
3453                }
3454                runtime
3455                    .active_turn_selections
3456                    .write()
3457                    .await
3458                    .remove(&turn_id_for_task);
3459                runtime
3460                    .active_turn_contexts
3461                    .write()
3462                    .await
3463                    .remove(&turn_id_for_task);
3464                if !completed {
3465                    // Completion paths above consume registered provider cleanup
3466                    // handles. A setup failure before an engine can stream has no
3467                    // such handle; this is a harmless final defensive sweep.
3468                    let _ = runtime.provider_turn_cleanups.lock().map(|mut cleanups| {
3469                        cleanups.remove(&turn_id_for_task);
3470                    });
3471                }
3472                runtime.active_turns_changed.notify_waiters();
3473                if completed {
3474                    let _ = runtime
3475                        .continue_active_goal_after_turn(thread_id_for_task)
3476                        .await;
3477                }
3478            });
3479            drop(turn_admission);
3480            Ok(turn_id)
3481        })
3482    }
3483
3484    pub(crate) async fn has_active_turn_for_thread(&self, thread_id: &ThreadId) -> bool {
3485        self.active_turns
3486            .read()
3487            .await
3488            .values()
3489            .any(|handle| &handle.thread_id == thread_id)
3490    }
3491
3492    async fn ensure_codex_v2_team_capacity(
3493        &self,
3494        team: &TeamState,
3495        resuming_member_id: &str,
3496        candidate_model: Option<&str>,
3497    ) -> anyhow::Result<()> {
3498        if !is_codex_v2_team(team)
3499            && !candidate_model
3500                .is_some_and(|model| model_supports_reasoning_effort(model, REASONING_ULTRA))
3501        {
3502            return Ok(());
3503        }
3504        let active_thread_ids = self
3505            .active_turns
3506            .read()
3507            .await
3508            .values()
3509            .map(|handle| handle.thread_id.clone())
3510            .collect::<std::collections::HashSet<_>>();
3511        let resident_threads = 1 + team
3512            .members
3513            .iter()
3514            .filter(|member| {
3515                member.role != roder_api::teams::TeamMemberRole::Lead
3516                    && member.id != resuming_member_id
3517                    && active_thread_ids.contains(&member.thread_id)
3518            })
3519            .count();
3520        anyhow::ensure!(
3521            resident_threads < codex_v2::CODEX_V2_MAX_RESIDENT_TEAM_THREADS,
3522            "agent thread limit reached for this Codex V2 team: maximum {} resident threads (the lead plus 3 running subagents)",
3523            codex_v2::CODEX_V2_MAX_RESIDENT_TEAM_THREADS
3524        );
3525        Ok(())
3526    }
3527
3528    /// Number of currently running turns across all threads. Hosted runtime
3529    /// pools use this to avoid evicting tenants with active work.
3530    pub async fn active_turn_count(&self) -> usize {
3531        self.active_turns.read().await.len()
3532    }
3533
3534    pub async fn active_turn_for_thread(&self, thread_id: &ThreadId) -> Option<TurnId> {
3535        self.active_turns
3536            .read()
3537            .await
3538            .iter()
3539            .find_map(|(turn_id, handle)| (&handle.thread_id == thread_id).then(|| turn_id.clone()))
3540    }
3541
3542    pub async fn thread_activity(&self, thread_id: &ThreadId) -> ThreadActivity {
3543        let active_turn_id = self.active_turn_for_thread(thread_id).await;
3544        let draining_turn_id = if active_turn_id.is_none() {
3545            self.turn_drains
3546                .read()
3547                .await
3548                .iter()
3549                .find_map(|(turn_id, drain)| {
3550                    (&drain.thread_id == thread_id).then(|| turn_id.clone())
3551                })
3552        } else {
3553            None
3554        };
3555        let Some(active_turn_id) = active_turn_id.or(draining_turn_id) else {
3556            return ThreadActivity::default();
3557        };
3558
3559        let mut active_flags = Vec::new();
3560        {
3561            let pending_approvals = self.pending_tool_approvals.lock().await;
3562            if pending_approvals
3563                .values()
3564                .any(|pending| &pending.thread_id == thread_id && pending.turn_id == active_turn_id)
3565            {
3566                active_flags.push("approvalRequired".to_string());
3567            }
3568        }
3569        {
3570            let pending_inputs = self.pending_user_inputs.lock().await;
3571            if pending_inputs
3572                .values()
3573                .any(|pending| &pending.thread_id == thread_id && pending.turn_id == active_turn_id)
3574            {
3575                active_flags.push("userInputRequired".to_string());
3576            }
3577        }
3578        {
3579            let pending_external = self.pending_external_tool_calls.lock().await;
3580            if pending_external
3581                .values()
3582                .any(|pending| &pending.thread_id == thread_id && pending.turn_id == active_turn_id)
3583            {
3584                active_flags.push("externalToolPending".to_string());
3585            }
3586        }
3587        let interrupting = self
3588            .active_turns
3589            .read()
3590            .await
3591            .get(&active_turn_id)
3592            .is_some_and(|handle| handle.drain.interrupt_requested.load(Ordering::Acquire))
3593            || self
3594                .turn_drains
3595                .read()
3596                .await
3597                .get(&active_turn_id)
3598                .is_some_and(|drain| drain.interrupt_requested.load(Ordering::Acquire));
3599        if interrupting {
3600            active_flags.push("interrupting".to_string());
3601        }
3602        if self.pending_plan_exit().await.is_some_and(|pending| {
3603            &pending.thread_id == thread_id && pending.turn_id == active_turn_id
3604        }) {
3605            active_flags.push("planExitRequired".to_string());
3606        }
3607
3608        ThreadActivity {
3609            active_turn_id: Some(active_turn_id),
3610            active_flags,
3611        }
3612    }
3613
3614    pub async fn interrupt_turn(&self, thread_id: ThreadId, turn_id: TurnId) -> anyhow::Result<()> {
3615        self.interrupt_turn_with_reason(thread_id, turn_id, TurnLifecycleReason::UserInterrupt)
3616            .await
3617    }
3618
3619    async fn interrupt_turn_with_reason(
3620        &self,
3621        thread_id: ThreadId,
3622        turn_id: TurnId,
3623        reason: TurnLifecycleReason,
3624    ) -> anyhow::Result<()> {
3625        let handle = self.active_turns.write().await.remove(&turn_id);
3626        if let Some(handle) = handle {
3627            if handle
3628                .drain
3629                .interrupt_requested
3630                .swap(true, Ordering::AcqRel)
3631            {
3632                return Ok(());
3633            }
3634            *handle.drain.interrupt_reason.lock().await = Some(reason);
3635            self.turn_drains
3636                .write()
3637                .await
3638                .insert(turn_id.clone(), handle.drain.clone());
3639            let ownership = self.provider_cleanup_ownership(&turn_id);
3640            self.record_turn_lifecycle_with_ownership(
3641                thread_id.clone(),
3642                turn_id.clone(),
3643                TurnLifecycleState::InterruptRequested,
3644                TurnCleanupState::Requested,
3645                Some(reason),
3646                ownership,
3647            )
3648            .await;
3649            handle.abort.abort();
3650            self.active_turns_changed.notify_waiters();
3651        }
3652        self.active_turn_selections.write().await.remove(&turn_id);
3653        self.cancel_pending_external_tool_calls_for_turn(&turn_id)
3654            .await;
3655        Ok(())
3656    }
3657
3658    pub async fn steer_turn(
3659        &self,
3660        thread_id: ThreadId,
3661        turn_id: TurnId,
3662        message: String,
3663        images: Vec<InputImage>,
3664    ) -> anyhow::Result<()> {
3665        self.enqueue_turn_steer(thread_id, turn_id, message, images, None)
3666            .await
3667    }
3668
3669    pub(crate) async fn has_pending_turn_steers(&self, turn_id: &TurnId) -> bool {
3670        let active = self.active_turns.read().await.get(turn_id).cloned();
3671        let Some(active) = active else {
3672            return false;
3673        };
3674        let has_pending = !active.steers.lock().await.is_empty();
3675        has_pending
3676    }
3677
3678    async fn steer_turn_with_mailbox_ack(
3679        &self,
3680        thread_id: ThreadId,
3681        turn_id: TurnId,
3682        message: String,
3683        message_ids: Vec<String>,
3684        team_id: TeamId,
3685    ) -> anyhow::Result<()> {
3686        self.enqueue_turn_steer(
3687            thread_id,
3688            turn_id,
3689            message,
3690            Vec::new(),
3691            Some(MailboxDeliveryAck {
3692                team_id,
3693                message_ids,
3694            }),
3695        )
3696        .await
3697    }
3698
3699    async fn deliver_pending_team_mailbox(
3700        &self,
3701        team_id: &str,
3702        member_id: &str,
3703        member_thread_id: ThreadId,
3704        turn_id: TurnId,
3705    ) -> anyhow::Result<()> {
3706        let pending = self
3707            .teams
3708            .reserve_pending_mailbox_messages(team_id, member_id, &turn_id)
3709            .await?;
3710        if pending.is_empty() {
3711            return Ok(());
3712        }
3713        let message_ids = pending
3714            .iter()
3715            .map(|message| message.id.clone())
3716            .collect::<Vec<_>>();
3717        let team = self
3718            .read_team(team_id)
3719            .await
3720            .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
3721        if let Err(error) = self
3722            .steer_turn_with_mailbox_ack(
3723                member_thread_id,
3724                turn_id.clone(),
3725                format_mailbox_messages(&team, &pending),
3726                message_ids.clone(),
3727                team_id.to_string(),
3728            )
3729            .await
3730        {
3731            self.teams
3732                .release_mailbox_reservations(&turn_id, &message_ids)
3733                .await;
3734            return Err(error);
3735        }
3736        Ok(())
3737    }
3738
3739    async fn enqueue_turn_steer(
3740        &self,
3741        thread_id: ThreadId,
3742        turn_id: TurnId,
3743        message: String,
3744        images: Vec<InputImage>,
3745        mailbox_ack: Option<MailboxDeliveryAck>,
3746    ) -> anyhow::Result<()> {
3747        let message = message.trim().to_string();
3748        if message.is_empty() && images.is_empty() {
3749            return Ok(());
3750        }
3751
3752        let Some(active) = self.active_turns.read().await.get(&turn_id).cloned() else {
3753            anyhow::bail!("no active turn to steer");
3754        };
3755        active.steers.lock().await.push(QueuedTurnSteer {
3756            message: UserMessage::with_images(message.clone(), images),
3757            mailbox_ack,
3758        });
3759        self.emit(RoderEvent::TurnSteered(TurnSteered {
3760            thread_id,
3761            turn_id,
3762            message,
3763            timestamp: OffsetDateTime::now_utc(),
3764        }))
3765        .await;
3766        Ok(())
3767    }
3768
3769    pub async fn tool_specs(&self) -> Vec<roder_api::tools::ToolSpec> {
3770        let cfg = self.config.read().await;
3771        let model_profile =
3772            model_profile_for_provider_model(&cfg, &cfg.default_provider, &cfg.default_model);
3773        self.filtered_tool_specs(&cfg, &cfg.default_model, model_profile.as_ref(), &[], &[])
3774    }
3775
3776    pub fn subagent_definitions(&self) -> Vec<SubagentDefinition> {
3777        self.registry
3778            .subagent_dispatchers
3779            .iter()
3780            .flat_map(|dispatcher| dispatcher.definitions())
3781            .collect()
3782    }
3783
3784    async fn run_turn(
3785        self: &Arc<Self>,
3786        req: StartTurnRequest,
3787        turn_id: TurnId,
3788        initial_mailbox_ack: Option<MailboxDeliveryAck>,
3789    ) -> anyhow::Result<TurnRunOutcome> {
3790        let turn_started_at = OffsetDateTime::now_utc();
3791        self.emit(RoderEvent::TurnStarted(TurnStarted {
3792            thread_id: req.thread_id.clone(),
3793            turn_id: turn_id.clone(),
3794            runtime_profile: self.config.read().await.runtime_profile,
3795            timestamp: turn_started_at,
3796        }))
3797        .await;
3798        self.persist_turn_item(
3799            &req.thread_id,
3800            &turn_id,
3801            &TranscriptItem::UserMessage(UserMessage::with_images(
3802                req.message.clone(),
3803                req.images.clone(),
3804            )),
3805        )
3806        .await?;
3807        crate::hooks::run_lifecycle(
3808            self,
3809            &req.thread_id,
3810            &turn_id,
3811            Some(&req.workspace),
3812            "UserPromptSubmit",
3813            None,
3814            serde_json::json!({"prompt": req.message}),
3815        )
3816        .await;
3817        if let Some(ack) = initial_mailbox_ack {
3818            self.teams
3819                .mark_mailbox_messages_delivered(&ack.team_id, &turn_id, &ack.message_ids)
3820                .await?;
3821        }
3822
3823        let mut cfg = self.config.read().await.clone();
3824        let runtime_profile = cfg.runtime_profile;
3825        let turn_deadline = turn_deadline_for_config(&cfg);
3826        let deadline_finalization_reserve =
3827            crate::deadline_policy::finalization_reserve_seconds(cfg.turn_deadline_seconds);
3828        let selection_mode = self.selection_mode_for_thread(&req.thread_id).await?;
3829        let concrete_selection = selection_mode
3830            .as_ref()
3831            .map(ModelSelectionMode::concrete_selection);
3832        let default_provider = req
3833            .provider_override
3834            .clone()
3835            .or_else(|| {
3836                concrete_selection
3837                    .as_ref()
3838                    .map(|selection| selection.provider.clone())
3839            })
3840            .unwrap_or(cfg.default_provider.clone());
3841        let default_model = req
3842            .model_override
3843            .clone()
3844            .or_else(|| {
3845                concrete_selection
3846                    .as_ref()
3847                    .map(|selection| selection.model.clone())
3848            })
3849            .unwrap_or(cfg.default_model.clone());
3850        if let Some(reasoning) = req.reasoning_override.as_deref().or_else(|| {
3851            selection_mode
3852                .as_ref()
3853                .and_then(ModelSelectionMode::reasoning)
3854        }) {
3855            validate_reasoning_effort(&default_model, reasoning)?;
3856            cfg.reasoning = Some(reasoning.to_string());
3857        }
3858        let turn_has_concrete_model_override =
3859            req.provider_override.is_some() || req.model_override.is_some();
3860        let (turn_inference_router, turn_inference_router_profile) = match &selection_mode {
3861            Some(ModelSelectionMode::Auto {
3862                router_id, profile, ..
3863            }) if !turn_has_concrete_model_override => (
3864                RuntimeInferenceRouterConfig {
3865                    enabled: true,
3866                    router_id: Some(router_id.clone()),
3867                },
3868                profile.clone(),
3869            ),
3870            _ => (RuntimeInferenceRouterConfig::disabled(), None),
3871        };
3872        let mut provider = default_provider.clone();
3873        let mut model = default_model.clone();
3874        let mut model_profile = model_profile_for_provider_model(&cfg, &provider, &model);
3875        let workspace = req.workspace.clone();
3876        let mut transcript = self.transcript_for_turn(&req, &turn_id, &model).await?;
3877        let mut compacted_this_turn = transcript
3878            .iter()
3879            .any(crate::compaction::is_compaction_boundary);
3880        let effective_policy_mode = self.effective_policy_mode_for_thread(&req.thread_id).await;
3881        let agent_swarm_mode_active = self
3882            .effective_agent_swarm_mode_for_thread(&req.thread_id)
3883            .await;
3884        let ultra_mode_active = self.effective_ultra_mode_for_thread(&req.thread_id).await;
3885        let thread_overrides = self.thread_turn_overrides(&req.thread_id).await?;
3886        let mut final_assistant_text = String::new();
3887        let mut final_phase_messages = Vec::<AssistantMessage>::new();
3888        let mut final_reasoning_text = String::new();
3889        let mut final_provider_metadata = None;
3890        let mut exhausted_tool_rounds = true;
3891        let mut verification_gate =
3892            VerificationGateState::new(req.message.clone(), runtime_profile);
3893        let mut speed_policy = SpeedPolicyState::default();
3894        let mut reliability = TurnReliabilityState::default();
3895        let mut turn_usage = TokenUsage::default();
3896        // Overwritten on every inference step's Completed event so only the
3897        // terminal step's stop reason survives (mid-turn tool_use steps must
3898        // not leak onto turn/completed).
3899        let mut turn_finish_reason: Option<String> = None;
3900        let mut deadline_finalization_requested = false;
3901        let mut deadline_scoreable_completion_requested = false;
3902        let mut task_ledger_completion_reminders = 0_u8;
3903        let mut task_ledger_scoreable_checkpoints = 0_u8;
3904        let mut empty_tool_call_nudges_used = 0_u32;
3905        let mut provider_stream_retry_attempts = 0_u32;
3906        let mut context_limit_recovery_attempts = 0_u32;
3907        let mut routing_candidates = None;
3908        let routing_transcript_start = transcript.len().saturating_sub(1);
3909        let mut routing_escalations = 0_u32;
3910        let mut model_switch_summary_selection = None::<ModelSelection>;
3911
3912        'tool_rounds: for round_index in 0..MAX_TOOL_ROUNDS_PER_TURN {
3913            if let Some(deadline) = turn_deadline
3914                && deadline_expired(deadline)
3915            {
3916                self.fail_turn_due_to_deadline(&req.thread_id, &turn_id, deadline, &transcript)
3917                    .await?;
3918                return Ok(TurnRunOutcome::Stopped);
3919            }
3920            let steers = self.drain_turn_steers(&turn_id).await;
3921            self.append_steers(&req, &turn_id, &mut transcript, steers)
3922                .await?;
3923            if runtime_profile == RuntimeProfile::Eval
3924                && let Some(remaining) = crate::deadline_policy::should_start_finalization(
3925                    turn_deadline,
3926                    deadline_finalization_reserve,
3927                    deadline_finalization_requested || deadline_scoreable_completion_requested,
3928                )
3929            {
3930                if req.task_ledger_required
3931                    && task_ledger_completion_reminders < TASK_LEDGER_COMPLETION_REMINDER_LIMIT
3932                    && let Some(prompt) = task_ledger_completion_prompt(&transcript)
3933                {
3934                    task_ledger_completion_reminders += 1;
3935                    deadline_scoreable_completion_requested = true;
3936                    let item = TranscriptItem::UserMessage(UserMessage::text(
3937                        task_ledger_deadline_completion_prompt(
3938                            remaining,
3939                            deadline_finalization_reserve,
3940                            &prompt,
3941                        ),
3942                    ));
3943                    self.persist_turn_item(&req.thread_id, &turn_id, &item)
3944                        .await?;
3945                    transcript.push(item);
3946                    continue 'tool_rounds;
3947                } else {
3948                    self.start_deadline_finalization(
3949                        &req.thread_id,
3950                        &turn_id,
3951                        &mut transcript,
3952                        remaining,
3953                    )
3954                    .await?;
3955                    deadline_finalization_requested = true;
3956                }
3957            }
3958            if runtime_profile == RuntimeProfile::Eval
3959                && req.task_ledger_required
3960                && !deadline_finalization_requested
3961                && task_ledger_scoreable_checkpoints < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
3962                && let Some(remaining) = deadline_remaining_seconds(turn_deadline)
3963                && remaining <= TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS
3964                && remaining > deadline_finalization_reserve
3965                && let Some(prompt) = task_ledger_completion_prompt(&transcript)
3966            {
3967                task_ledger_scoreable_checkpoints += 1;
3968                let item = TranscriptItem::UserMessage(UserMessage::text(
3969                    task_ledger_scoreable_checkpoint_prompt(remaining, &prompt),
3970                ));
3971                self.persist_turn_item(&req.thread_id, &turn_id, &item)
3972                    .await?;
3973                transcript.push(item);
3974                continue 'tool_rounds;
3975            }
3976            if turn_inference_router.is_active() && routing_candidates.is_none() {
3977                routing_candidates =
3978                    Some(collect_inference_routing_candidates(&self.registry).await);
3979            }
3980            let routing_tools_model = model.clone();
3981            let routing_tools = self.filtered_tool_specs(
3982                &cfg,
3983                &model,
3984                model_profile.as_ref(),
3985                &thread_overrides.tool_allowlist,
3986                &thread_overrides.external_tools,
3987            );
3988            let prior_failures =
3989                transcript_failure_count_since(&transcript, routing_transcript_start)
3990                    .max(reliability.tool_failure_count())
3991                    .saturating_add(provider_stream_retry_attempts);
3992            let routing_selection = route_inference_selection(
3993                &self.registry,
3994                &turn_inference_router,
3995                InferenceRoutingRequest {
3996                    thread_id: &req.thread_id,
3997                    turn_id: &turn_id,
3998                    round_index: round_index as u32,
3999                    runtime_profile,
4000                    phase: speed_policy.phase(),
4001                    profile: turn_inference_router_profile.as_deref(),
4002                    default_selection: ModelSelection {
4003                        provider: default_provider.clone(),
4004                        model: default_model.clone(),
4005                    },
4006                    transcript: &transcript,
4007                    tools: &routing_tools,
4008                    candidates: routing_candidates.as_deref(),
4009                    prior_failures,
4010                    prior_escalations: routing_escalations,
4011                },
4012            )
4013            .await;
4014            if let Some(decision) = routing_selection.decision.clone() {
4015                if matches!(decision.outcome, InferenceRoutingOutcome::Escalated) {
4016                    routing_escalations = routing_escalations.saturating_add(1);
4017                }
4018                self.emit(RoderEvent::InferenceRoutingDecision(
4019                    InferenceRoutingDecisionEvent {
4020                        thread_id: req.thread_id.clone(),
4021                        turn_id: turn_id.clone(),
4022                        round_index: round_index as u32,
4023                        default_selection: ModelSelection {
4024                            provider: default_provider.clone(),
4025                            model: default_model.clone(),
4026                        },
4027                        selected_selection: routing_selection.selection.clone(),
4028                        decision,
4029                        timestamp: OffsetDateTime::now_utc(),
4030                    },
4031                ))
4032                .await;
4033            }
4034            provider = routing_selection.selection.provider.clone();
4035            model = routing_selection.selection.model.clone();
4036            let engine = self.engine_for(&provider)?;
4037            let capabilities = engine.capabilities();
4038            model_profile = model_profile_for_provider_model(&cfg, &provider, &model);
4039            let tools = if capabilities.tool_calls {
4040                if model == routing_tools_model {
4041                    routing_tools.clone()
4042                } else {
4043                    self.filtered_tool_specs(
4044                        &cfg,
4045                        &model,
4046                        model_profile.as_ref(),
4047                        &thread_overrides.tool_allowlist,
4048                        &thread_overrides.external_tools,
4049                    )
4050                }
4051            } else {
4052                Vec::new()
4053            };
4054            let parallel_tool_calls = parallel_tool_calls_for_model(&cfg, &model);
4055            let tool_choice = if tools.is_empty() {
4056                ToolChoice::None
4057            } else {
4058                ToolChoice::Auto
4059            };
4060            let summary_selection = ModelSelection {
4061                provider: provider.clone(),
4062                model: model.clone(),
4063            };
4064            if model_switch_summary_selection.as_ref() != Some(&summary_selection) {
4065                if let Some(summary) = model_switch_summary(
4066                    &transcript,
4067                    model_profile.as_ref(),
4068                    &provider,
4069                    &model,
4070                    &tools,
4071                ) {
4072                    let item = TranscriptItem::UserMessage(UserMessage::text(summary));
4073                    self.persist_turn_item(&req.thread_id, &turn_id, &item)
4074                        .await?;
4075                    transcript.push(item);
4076                }
4077                model_switch_summary_selection = Some(summary_selection);
4078            }
4079
4080            if !capabilities.image_input && transcript_has_images(&transcript) {
4081                self.fail_turn_with_error(
4082                    &req.thread_id,
4083                    &turn_id,
4084                    format!("provider {provider} does not support image input"),
4085                )
4086                .await?;
4087                return Ok(TurnRunOutcome::Stopped);
4088            }
4089            transcript = self
4090                .compact_transcript_if_needed(
4091                    &req.thread_id,
4092                    &turn_id,
4093                    &provider,
4094                    &model,
4095                    transcript,
4096                    self.compaction_options_for_turn(&req.thread_id, !compacted_this_turn),
4097                )
4098                .await?;
4099            compacted_this_turn = compacted_this_turn
4100                || transcript
4101                    .iter()
4102                    .any(crate::compaction::is_compaction_boundary);
4103
4104            let speed_policy_decision =
4105                speed_policy.decision(runtime_profile, &model, &cfg.speed_policy);
4106            let request_reasoning = reasoning_from_decision(
4107                speed_policy_decision.as_ref(),
4108                routing_selection
4109                    .reasoning
4110                    .clone()
4111                    .unwrap_or_else(|| reasoning_for_model(&cfg, &model)),
4112            );
4113            self.active_turn_selections.write().await.insert(
4114                turn_id.clone(),
4115                ModelSelectionMode::manual(
4116                    provider.clone(),
4117                    model.clone(),
4118                    request_reasoning.level.clone(),
4119                ),
4120            );
4121            if let Some(limit) = reliability.record_model_call(
4122                &cfg.reliability,
4123                runtime_profile == RuntimeProfile::Interactive,
4124            ) {
4125                self.fail_turn_due_to_reliability_limit(
4126                    &req.thread_id,
4127                    &turn_id,
4128                    &provider,
4129                    &model,
4130                    limit,
4131                    &transcript,
4132                )
4133                .await?;
4134                return Ok(TurnRunOutcome::Stopped);
4135            }
4136            self.emit(RoderEvent::InferenceStarted(InferenceStarted {
4137                thread_id: req.thread_id.clone(),
4138                turn_id: turn_id.clone(),
4139                engine_id: engine.id(),
4140                model: ModelSelection {
4141                    provider: provider.clone(),
4142                    model: model.clone(),
4143                },
4144                reasoning: request_reasoning.clone(),
4145                speed_policy: speed_policy_decision.clone(),
4146                deadline_remaining_seconds: deadline_remaining_seconds(turn_deadline),
4147                timestamp: OffsetDateTime::now_utc(),
4148            }))
4149            .await;
4150
4151            let mut instructions = req.instructions.clone();
4152            if let Some(extra) = &thread_overrides.developer_instructions {
4153                instructions = apply_thread_developer_instructions(instructions, extra);
4154            }
4155            if let Some(context) = req.developer_context.as_deref() {
4156                instructions = apply_turn_developer_context(instructions, context);
4157            }
4158            let mut instructions = apply_runtime_profile(instructions, runtime_profile);
4159            if let Some(profile) = &model_profile {
4160                instructions = apply_model_instruction_overlay(instructions, profile);
4161            }
4162            if req.task_ledger_required
4163                && runtime_profile == RuntimeProfile::Eval
4164                && !transcript_has_task_ledger(&transcript)
4165            {
4166                instructions = apply_task_ledger_required(instructions);
4167            }
4168            if effective_policy_mode == PolicyMode::Plan {
4169                instructions = apply_plan_mode(instructions);
4170            }
4171            if agent_swarm_mode_active {
4172                instructions = apply_agent_swarm_mode(instructions);
4173            }
4174            // Ultra mode (any model) enables proactive multi-agent policy.
4175            // Codex Sol/Terra Ultra effort still does too without requiring the
4176            // mode flag. Models that advertise Ultra effort keep the
4177            // explicit-request-only policy on lower efforts when ultra mode is
4178            // off; other models get multi-agent policy only when ultra mode is on.
4179            let model_has_ultra_effort = model_supports_reasoning_effort(&model, REASONING_ULTRA);
4180            let effort_is_ultra = request_reasoning.level.as_deref() == Some(REASONING_ULTRA);
4181            let proactive_multi_agent = ultra_mode_active || effort_is_ultra;
4182            if ultra_mode_active || model_has_ultra_effort {
4183                instructions = apply_codex_multi_agent_mode(instructions, proactive_multi_agent);
4184            }
4185            instructions = self
4186                .goals
4187                .apply_goal_instructions(&req.thread_id, instructions)
4188                .await?;
4189            instructions =
4190                apply_parallel_web_tools(instructions, tools.iter().map(|spec| spec.name.as_str()));
4191            let mut request_metadata = serde_json::json!({});
4192            if let Some(decision) = &speed_policy_decision {
4193                request_metadata["speedPolicy"] = serde_json::json!(decision);
4194            }
4195            if let Some(decision) = routing_selection.decision.as_ref() {
4196                request_metadata["inferenceRouting"] = serde_json::json!(decision);
4197            }
4198            if let Some(remaining) = deadline_remaining_seconds(turn_deadline) {
4199                request_metadata["deadlineRemainingSeconds"] = serde_json::json!(remaining);
4200            }
4201            if let Some(profile) = &model_profile {
4202                request_metadata["modelProfile"] = serde_json::json!({
4203                    "model": profile.model,
4204                    "providerFamily": profile.provider_family,
4205                    "editTool": profile.edit_tool,
4206                    "schemaPolicy": profile.schema_policy,
4207                    "instructionOverlay": profile.instruction_overlay,
4208                    "parallelToolCalls": profile.parallel_tool_calls,
4209                    "autoCompactTokenLimit": profile.auto_compact_token_limit,
4210                });
4211            }
4212            let task_ledger_required_this_round = req.task_ledger_required
4213                && runtime_profile == RuntimeProfile::Eval
4214                && !deadline_finalization_requested
4215                && !transcript_has_task_ledger(&transcript);
4216            let task_ledger_tools = (capabilities.tool_calls && task_ledger_required_this_round)
4217                .then(|| self.task_ledger_tool_specs(model_profile.as_ref()))
4218                .filter(|tools| !tools.is_empty());
4219            let request_tools = if deadline_finalization_requested {
4220                Vec::new()
4221            } else if let Some(ledger_tools) = &task_ledger_tools {
4222                ledger_tools.clone()
4223            } else {
4224                tools.clone()
4225            };
4226            let request_tool_choice = if deadline_finalization_requested {
4227                ToolChoice::None
4228            } else if task_ledger_tools.is_some() {
4229                ToolChoice::Specific(TASK_LEDGER_TOOL_NAME.to_string())
4230            } else {
4231                tool_choice.clone()
4232            };
4233            if deadline_finalization_requested {
4234                request_metadata["deadlineFinalization"] = serde_json::json!({
4235                    "reserveSeconds": deadline_finalization_reserve,
4236                    "remainingSeconds": deadline_remaining_seconds(turn_deadline),
4237                });
4238            } else if deadline_scoreable_completion_requested {
4239                request_metadata["deadlineScoreableCompletion"] = serde_json::json!({
4240                    "reserveSeconds": deadline_finalization_reserve,
4241                    "remainingSeconds": deadline_remaining_seconds(turn_deadline),
4242                });
4243            }
4244            let request = AgentInferenceRequest {
4245                model: ModelSelection {
4246                    provider: provider.clone(),
4247                    model: model.clone(),
4248                },
4249                instructions,
4250                transcript: transcript.clone(),
4251                tools: request_tools,
4252                tool_choice: request_tool_choice,
4253                reasoning: request_reasoning,
4254                output: OutputConfig::default(),
4255                runtime: RuntimeHints {
4256                    auto_compact_token_limit: server_side_compaction_threshold(&cfg, &model),
4257                    profile: runtime_profile,
4258                    parallel_tool_calls: Some(parallel_tool_calls),
4259                    hosted_web_search: cfg.hosted_web_search.clone(),
4260                    tool_search: tool_search_for_provider_model(&cfg, &provider, &model),
4261                    speed_policy: speed_policy_decision,
4262                    reliability: Some(cfg.reliability.clone().into()),
4263                    deadline_remaining_seconds: deadline_remaining_seconds(turn_deadline),
4264                    service_tier: req.service_tier_override.clone(),
4265                    ..RuntimeHints::default()
4266                },
4267                metadata: request_metadata,
4268            };
4269
4270            let ctx = InferenceTurnContext {
4271                thread_id: &req.thread_id,
4272                turn_id: &turn_id,
4273                tool_executor: Some(std::sync::Arc::new(
4274                    crate::tool_execution::RuntimeTurnToolExecutor {
4275                        runtime: Arc::clone(self),
4276                        thread_id: req.thread_id.clone(),
4277                        turn_id: turn_id.clone(),
4278                        workspace: Some(workspace.clone()),
4279                        deadline: turn_deadline,
4280                    },
4281                )),
4282            };
4283            let stream_future = engine.stream_turn(ctx, request);
4284            let mut stream = if let Some((deadline, timeout_action)) = inference_timeout_deadline(
4285                turn_deadline,
4286                runtime_profile,
4287                req.task_ledger_required,
4288                deadline_finalization_reserve,
4289                deadline_finalization_requested || deadline_scoreable_completion_requested,
4290                task_ledger_scoreable_checkpoints,
4291                &transcript,
4292            ) {
4293                match tokio::time::timeout_at(deadline_instant(deadline), stream_future).await {
4294                    Ok(stream) => match stream {
4295                        Ok(stream) => stream,
4296                        Err(err) => {
4297                            let error = err.to_string();
4298                            if self
4299                                .try_recover_from_context_limit(
4300                                    &req.thread_id,
4301                                    &turn_id,
4302                                    &provider,
4303                                    &model,
4304                                    &error,
4305                                    &mut transcript,
4306                                    &mut compacted_this_turn,
4307                                    &mut context_limit_recovery_attempts,
4308                                )
4309                                .await?
4310                            {
4311                                continue 'tool_rounds;
4312                            }
4313                            return Err(err);
4314                        }
4315                    },
4316                    Err(_) => {
4317                        if runtime_profile == RuntimeProfile::Eval
4318                            && !deadline_finalization_requested
4319                        {
4320                            let remaining = deadline_remaining_seconds(turn_deadline).unwrap_or(0);
4321                            if timeout_action == InferenceTimeoutAction::ScoreableCheckpoint
4322                                && task_ledger_scoreable_checkpoints
4323                                    < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
4324                                && let Some(prompt) = task_ledger_completion_prompt(&transcript)
4325                            {
4326                                task_ledger_scoreable_checkpoints += 1;
4327                                let item = TranscriptItem::UserMessage(UserMessage::text(
4328                                    task_ledger_scoreable_checkpoint_prompt(remaining, &prompt),
4329                                ));
4330                                self.persist_turn_item(&req.thread_id, &turn_id, &item)
4331                                    .await?;
4332                                transcript.push(item);
4333                                continue 'tool_rounds;
4334                            }
4335                            self.start_deadline_finalization(
4336                                &req.thread_id,
4337                                &turn_id,
4338                                &mut transcript,
4339                                remaining,
4340                            )
4341                            .await?;
4342                            deadline_finalization_requested = true;
4343                            continue 'tool_rounds;
4344                        }
4345                        self.fail_turn_due_to_deadline(
4346                            &req.thread_id,
4347                            &turn_id,
4348                            deadline,
4349                            &transcript,
4350                        )
4351                        .await?;
4352                        return Ok(TurnRunOutcome::Stopped);
4353                    }
4354                }
4355            } else {
4356                match stream_future.await {
4357                    Ok(stream) => stream,
4358                    Err(err) => {
4359                        let error = err.to_string();
4360                        if self
4361                            .try_recover_from_context_limit(
4362                                &req.thread_id,
4363                                &turn_id,
4364                                &provider,
4365                                &model,
4366                                &error,
4367                                &mut transcript,
4368                                &mut compacted_this_turn,
4369                                &mut context_limit_recovery_attempts,
4370                            )
4371                            .await?
4372                        {
4373                            continue 'tool_rounds;
4374                        }
4375                        return Err(err);
4376                    }
4377                }
4378            };
4379            let mut assistant_text = String::new();
4380            let mut phase_messages = Vec::<AssistantMessage>::new();
4381            let mut reasoning_text = String::new();
4382            let mut tool_calls = Vec::new();
4383            let mut provider_metadata = None;
4384            // Captured from mid-stream Compaction(completed) events. Codex/OpenAI
4385            // sometimes abort the SSE stream after emitting the compaction item
4386            // ("error decoding response body") before response.completed, so we
4387            // must not rely solely on ProviderMetadata for the boundary.
4388            let mut stream_compaction_item: Option<serde_json::Value> = None;
4389
4390            loop {
4391                let next = if let Some((deadline, timeout_action)) = inference_timeout_deadline(
4392                    turn_deadline,
4393                    runtime_profile,
4394                    req.task_ledger_required,
4395                    deadline_finalization_reserve,
4396                    deadline_finalization_requested || deadline_scoreable_completion_requested,
4397                    task_ledger_scoreable_checkpoints,
4398                    &transcript,
4399                ) {
4400                    match tokio::time::timeout_at(deadline_instant(deadline), stream.next()).await {
4401                        Ok(next) => next,
4402                        Err(_) => {
4403                            if runtime_profile == RuntimeProfile::Eval
4404                                && !deadline_finalization_requested
4405                            {
4406                                let remaining =
4407                                    deadline_remaining_seconds(turn_deadline).unwrap_or(0);
4408                                if timeout_action == InferenceTimeoutAction::ScoreableCheckpoint
4409                                    && task_ledger_scoreable_checkpoints
4410                                        < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
4411                                    && let Some(prompt) = task_ledger_completion_prompt(&transcript)
4412                                {
4413                                    task_ledger_scoreable_checkpoints += 1;
4414                                    let item = TranscriptItem::UserMessage(UserMessage::text(
4415                                        task_ledger_scoreable_checkpoint_prompt(remaining, &prompt),
4416                                    ));
4417                                    self.persist_turn_item(&req.thread_id, &turn_id, &item)
4418                                        .await?;
4419                                    transcript.push(item);
4420                                    continue 'tool_rounds;
4421                                }
4422                                self.start_deadline_finalization(
4423                                    &req.thread_id,
4424                                    &turn_id,
4425                                    &mut transcript,
4426                                    remaining,
4427                                )
4428                                .await?;
4429                                deadline_finalization_requested = true;
4430                                continue 'tool_rounds;
4431                            }
4432                            self.fail_turn_due_to_deadline(
4433                                &req.thread_id,
4434                                &turn_id,
4435                                deadline,
4436                                &transcript,
4437                            )
4438                            .await?;
4439                            return Ok(TurnRunOutcome::Stopped);
4440                        }
4441                    }
4442                } else {
4443                    stream.next().await
4444                };
4445                let Some(res) = next else {
4446                    break;
4447                };
4448                let event = match res {
4449                    Ok(event) => event,
4450                    Err(err) => {
4451                        let error = err.to_string();
4452                        if runtime_profile == RuntimeProfile::Eval
4453                            && !deadline_finalization_requested
4454                            && let Some(cause) = provider_stream_retry_cause(&error)
4455                        {
4456                            let retry_attempt = provider_stream_retry_attempts.saturating_add(1);
4457                            let policy: ReliabilityRequestPolicy = cfg.reliability.clone().into();
4458                            if retry_attempt < policy.provider_retry_max_attempts {
4459                                provider_stream_retry_attempts = retry_attempt;
4460                                let delay_ms = provider_retry_delay_ms(&policy, retry_attempt);
4461                                self.emit(RoderEvent::ReliabilityRetryRecorded(
4462                                    ReliabilityRetryRecorded {
4463                                        context: ReliabilityContext {
4464                                            thread_id: req.thread_id.clone(),
4465                                            turn_id: turn_id.clone(),
4466                                            provider: Some(provider.clone()),
4467                                            model: Some(model.clone()),
4468                                            ..ReliabilityContext::default()
4469                                        },
4470                                        error_class: ReliabilityErrorClass::ProviderError,
4471                                        decision: ReliabilityRetryDecision::Retry,
4472                                        attempt: retry_attempt,
4473                                        max_attempts: policy.provider_retry_max_attempts,
4474                                        delay_ms: Some(delay_ms),
4475                                        details: ReliabilityDetails::redacted(format!(
4476                                            "{cause}: {error}"
4477                                        )),
4478                                        timestamp: OffsetDateTime::now_utc(),
4479                                    },
4480                                ))
4481                                .await;
4482                                if delay_ms > 0 {
4483                                    tokio::time::sleep(std::time::Duration::from_millis(delay_ms))
4484                                        .await;
4485                                }
4486                                continue 'tool_rounds;
4487                            }
4488                        }
4489                        if self
4490                            .try_recover_from_context_limit(
4491                                &req.thread_id,
4492                                &turn_id,
4493                                &provider,
4494                                &model,
4495                                &error,
4496                                &mut transcript,
4497                                &mut compacted_this_turn,
4498                                &mut context_limit_recovery_attempts,
4499                            )
4500                            .await?
4501                        {
4502                            continue 'tool_rounds;
4503                        }
4504                        self.emit(RoderEvent::TurnFailed(TurnFailed {
4505                            thread_id: req.thread_id.clone(),
4506                            turn_id: turn_id.clone(),
4507                            error: error.clone(),
4508                            error_kind: None,
4509                            usage: None,
4510                            timestamp: OffsetDateTime::now_utc(),
4511                        }))
4512                        .await;
4513                        self.complete_team_member_turn_with_result(
4514                            &req.thread_id,
4515                            &turn_id,
4516                            TeamMemberStatus::Failed,
4517                            (!assistant_text.trim().is_empty()).then(|| assistant_text.clone()),
4518                            Some(error),
4519                        )
4520                        .await?;
4521                        return Err(err);
4522                    }
4523                };
4524
4525                let inference_timestamp = OffsetDateTime::now_utc();
4526                self.emit(RoderEvent::InferenceEventReceived(InferenceEventReceived {
4527                    thread_id: req.thread_id.clone(),
4528                    turn_id: turn_id.clone(),
4529                    event: event.clone(),
4530                    timestamp: inference_timestamp,
4531                }))
4532                .await;
4533
4534                match event {
4535                    InferenceEvent::MessageDelta(delta) => {
4536                        if let Some((team_id, member)) =
4537                            self.teams.member_for_thread(&req.thread_id).await
4538                        {
4539                            self.emit(RoderEvent::TeamMemberMessageDelta(TeamMemberMessageDelta {
4540                                team_id,
4541                                member_id: member.id,
4542                                member_thread_id: req.thread_id.clone(),
4543                                turn_id: turn_id.clone(),
4544                                delta: delta.text.clone(),
4545                                timestamp: OffsetDateTime::now_utc(),
4546                            }))
4547                            .await;
4548                        }
4549                        if is_final_answer_phase(delta.phase.as_deref()) {
4550                            assistant_text.push_str(&delta.text);
4551                        } else if let Some(last) = phase_messages.last_mut()
4552                            && last.phase == delta.phase
4553                        {
4554                            last.text.push_str(&delta.text);
4555                        } else {
4556                            phase_messages.push(AssistantMessage {
4557                                text: delta.text,
4558                                phase: delta.phase,
4559                            });
4560                        }
4561                    }
4562                    InferenceEvent::ReasoningDelta(delta) => reasoning_text.push_str(&delta.text),
4563                    InferenceEvent::ToolCallCompleted(call) => tool_calls.push(call),
4564                    InferenceEvent::Failed(failure) => {
4565                        speed_policy.record_failure();
4566                        let error = failure.message;
4567                        if self
4568                            .try_recover_from_context_limit(
4569                                &req.thread_id,
4570                                &turn_id,
4571                                &provider,
4572                                &model,
4573                                &error,
4574                                &mut transcript,
4575                                &mut compacted_this_turn,
4576                                &mut context_limit_recovery_attempts,
4577                            )
4578                            .await?
4579                        {
4580                            continue 'tool_rounds;
4581                        }
4582                        self.persist_turn_item(
4583                            &req.thread_id,
4584                            &turn_id,
4585                            &TranscriptItem::Error(ErrorRecord {
4586                                message: error.clone(),
4587                            }),
4588                        )
4589                        .await?;
4590                        self.emit(RoderEvent::TurnFailed(TurnFailed {
4591                            thread_id: req.thread_id.clone(),
4592                            turn_id: turn_id.clone(),
4593                            error: error.clone(),
4594                            error_kind: None,
4595                            usage: None,
4596                            timestamp: OffsetDateTime::now_utc(),
4597                        }))
4598                        .await;
4599                        self.complete_team_member_turn_with_result(
4600                            &req.thread_id,
4601                            &turn_id,
4602                            TeamMemberStatus::Failed,
4603                            (!assistant_text.trim().is_empty()).then(|| assistant_text.clone()),
4604                            Some(error),
4605                        )
4606                        .await?;
4607                        return Ok(TurnRunOutcome::Stopped);
4608                    }
4609                    InferenceEvent::Usage(usage) => {
4610                        turn_usage.add_assign(&usage);
4611                    }
4612                    InferenceEvent::Completed(metadata) => {
4613                        turn_finish_reason = metadata
4614                            .stop_reason
4615                            .as_deref()
4616                            .map(finish_reason_from_stop_reason);
4617                    }
4618                    InferenceEvent::Compaction(progress) => {
4619                        // Persist the boundary as soon as the provider emits a
4620                        // completed compaction item. ChatGPT Codex often aborts
4621                        // the SSE stream right after ("error decoding response
4622                        // body"), so waiting for ProviderMetadata loses the
4623                        // boundary and every subsequent request re-compacts.
4624                        if progress.status == "completed"
4625                            && let Some(item) = progress.item
4626                        {
4627                            let already = stream_compaction_item
4628                                .as_ref()
4629                                .and_then(|existing| {
4630                                    existing.get("id").and_then(serde_json::Value::as_str)
4631                                })
4632                                .zip(item.get("id").and_then(serde_json::Value::as_str))
4633                                .is_some_and(|(a, b)| a == b);
4634                            if !already {
4635                                stream_compaction_item = Some(item.clone());
4636                                let boundary = TranscriptItem::ProviderMetadata(
4637                                    crate::compaction::provider_metadata_from_compaction_item(item),
4638                                );
4639                                self.persist_turn_item(&req.thread_id, &turn_id, &boundary)
4640                                    .await?;
4641                                transcript.push(boundary);
4642                                transcript =
4643                                    crate::compaction::trim_to_last_compaction_boundary(transcript);
4644                                compacted_this_turn = true;
4645                            }
4646                        }
4647                    }
4648                    InferenceEvent::HostedToolCallStarted(_)
4649                    | InferenceEvent::HostedToolCallCompleted(_)
4650                    | InferenceEvent::ToolCallStarted(_)
4651                    | InferenceEvent::ToolCallDelta(_) => {}
4652                    InferenceEvent::ProviderMetadata(mut metadata) => {
4653                        if let Some(compaction_item) = stream_compaction_item.as_ref() {
4654                            let _ = crate::compaction::ensure_provider_metadata_compaction(
4655                                &mut metadata,
4656                                compaction_item,
4657                            );
4658                        }
4659                        provider_metadata = Some(metadata);
4660                    }
4661                }
4662            }
4663
4664            speed_policy.record_model_output(
4665                !assistant_text.is_empty() || !phase_messages.is_empty(),
4666                tool_calls.len(),
4667            );
4668            if tool_calls.is_empty() {
4669                let steers = self.drain_turn_steers(&turn_id).await;
4670                if !steers.is_empty() {
4671                    for message in phase_messages {
4672                        let item = TranscriptItem::AssistantMessage(message);
4673                        self.persist_turn_item(&req.thread_id, &turn_id, &item)
4674                            .await?;
4675                        transcript.push(item);
4676                        self.persist_model_profile_segment(
4677                            &req.thread_id,
4678                            &turn_id,
4679                            model_profile.as_ref(),
4680                            &provider,
4681                            &model,
4682                            "assistant",
4683                        )
4684                        .await?;
4685                    }
4686                    if !assistant_text.is_empty() {
4687                        let assistant = TranscriptItem::AssistantMessage(AssistantMessage {
4688                            text: assistant_text,
4689                            phase: Some(FINAL_ANSWER_PHASE.to_string()),
4690                        });
4691                        self.persist_turn_item(&req.thread_id, &turn_id, &assistant)
4692                            .await?;
4693                        transcript.push(assistant);
4694                        self.persist_model_profile_segment(
4695                            &req.thread_id,
4696                            &turn_id,
4697                            model_profile.as_ref(),
4698                            &provider,
4699                            &model,
4700                            "assistant",
4701                        )
4702                        .await?;
4703                    }
4704                    if let Some(metadata) = provider_metadata {
4705                        let had_provider_compaction =
4706                            crate::compaction::provider_metadata_has_compaction(&metadata);
4707                        let item = TranscriptItem::ProviderMetadata(metadata);
4708                        self.persist_turn_item(&req.thread_id, &turn_id, &item)
4709                            .await?;
4710                        transcript.push(item);
4711                        if had_provider_compaction {
4712                            // Server-side compaction replaced the prior window;
4713                            // drop pre-boundary items so the next request does
4714                            // not re-send (and re-compact) the full history.
4715                            transcript =
4716                                crate::compaction::trim_to_last_compaction_boundary(transcript);
4717                            compacted_this_turn = true;
4718                        }
4719                    }
4720                    self.append_steers(&req, &turn_id, &mut transcript, steers)
4721                        .await?;
4722                    continue;
4723                }
4724                if !deadline_finalization_requested
4725                    && req.task_ledger_required
4726                    && runtime_profile == RuntimeProfile::Eval
4727                    && task_ledger_completion_reminders < TASK_LEDGER_COMPLETION_REMINDER_LIMIT
4728                    && (!assistant_text.trim().is_empty() || !phase_messages.is_empty())
4729                    && let Some(prompt) = task_ledger_completion_prompt(&transcript)
4730                {
4731                    task_ledger_completion_reminders += 1;
4732                    let item = TranscriptItem::UserMessage(UserMessage::text(prompt));
4733                    self.persist_turn_item(&req.thread_id, &turn_id, &item)
4734                        .await?;
4735                    transcript.push(item);
4736                    continue;
4737                }
4738                if !deadline_finalization_requested
4739                    && let Some(prompt) = verification_gate.blocking_prompt()
4740                {
4741                    speed_policy.record_verification_required();
4742                    self.emit(RoderEvent::VerificationRequired(VerificationRequired {
4743                        thread_id: req.thread_id.clone(),
4744                        turn_id: turn_id.clone(),
4745                        reason: verification_gate.reason(),
4746                        changed_files: verification_gate.changed_files(),
4747                        tool_evidence: verification_gate.tool_evidence.clone(),
4748                        tests_run: verification_gate.tests_run.clone(),
4749                        open_gaps: verification_gate.open_gaps.clone(),
4750                        timestamp: OffsetDateTime::now_utc(),
4751                    }))
4752                    .await;
4753                    let item = TranscriptItem::UserMessage(UserMessage::text(prompt));
4754                    self.persist_turn_item(&req.thread_id, &turn_id, &item)
4755                        .await?;
4756                    transcript.push(item);
4757                    continue;
4758                }
4759                // Loop persistence nudge: in non-interactive/eval runs, when the
4760                // model returns a final message with no tool calls, gently prompt
4761                // it to keep going (bounded by `empty_tool_call_nudges`) before
4762                // ending the turn. Gated to non-interactive/eval so interactive
4763                // chat completions are never delayed, and only fires when the
4764                // model actually produced a final answer to re-examine.
4765                if !deadline_finalization_requested
4766                    && runtime_profile != RuntimeProfile::Interactive
4767                    && cfg.reliability.empty_tool_call_nudges > 0
4768                    && empty_tool_call_nudges_used < cfg.reliability.empty_tool_call_nudges
4769                    && (!assistant_text.trim().is_empty() || !phase_messages.is_empty())
4770                {
4771                    empty_tool_call_nudges_used += 1;
4772                    let item = TranscriptItem::UserMessage(UserMessage::text(
4773                        RELIABILITY_CONTINUATION_PROMPT.to_string(),
4774                    ));
4775                    self.persist_turn_item(&req.thread_id, &turn_id, &item)
4776                        .await?;
4777                    transcript.push(item);
4778                    continue;
4779                }
4780                if deadline_finalization_requested
4781                    && assistant_text.trim().is_empty()
4782                    && phase_messages.is_empty()
4783                {
4784                    assistant_text = format!(
4785                        "Deadline finalization completed without model text. {}",
4786                        turn_partial_result(&transcript)
4787                    );
4788                }
4789                final_phase_messages = phase_messages;
4790                final_assistant_text = assistant_text;
4791                final_reasoning_text = reasoning_text;
4792                final_provider_metadata = provider_metadata;
4793                exhausted_tool_rounds = false;
4794                break;
4795            }
4796
4797            for message in phase_messages {
4798                let item = TranscriptItem::AssistantMessage(message);
4799                self.persist_turn_item(&req.thread_id, &turn_id, &item)
4800                    .await?;
4801                transcript.push(item);
4802                self.persist_model_profile_segment(
4803                    &req.thread_id,
4804                    &turn_id,
4805                    model_profile.as_ref(),
4806                    &provider,
4807                    &model,
4808                    "assistant",
4809                )
4810                .await?;
4811            }
4812            if !assistant_text.is_empty() {
4813                transcript.push(TranscriptItem::AssistantMessage(AssistantMessage {
4814                    text: assistant_text,
4815                    phase: Some(FINAL_ANSWER_PHASE.to_string()),
4816                }));
4817                self.persist_model_profile_segment(
4818                    &req.thread_id,
4819                    &turn_id,
4820                    model_profile.as_ref(),
4821                    &provider,
4822                    &model,
4823                    "assistant",
4824                )
4825                .await?;
4826            }
4827            if let Some(metadata) = provider_metadata {
4828                let had_provider_compaction =
4829                    crate::compaction::provider_metadata_has_compaction(&metadata);
4830                let item = TranscriptItem::ProviderMetadata(metadata);
4831                self.persist_turn_item(&req.thread_id, &turn_id, &item)
4832                    .await?;
4833                transcript.push(item);
4834                if had_provider_compaction {
4835                    // Server-side compaction replaced the prior window; drop
4836                    // pre-boundary items so subsequent tool rounds use the
4837                    // compact window as the next input.
4838                    transcript = crate::compaction::trim_to_last_compaction_boundary(transcript);
4839                    compacted_this_turn = true;
4840                }
4841            }
4842            // Persist CoT before tool calls so providers that require
4843            // `reasoning_content` on tool-call assistant turns (DeepSeek) can
4844            // replay it on the next request. Without this, multi-step tool
4845            // rollouts drop the thinking block and the API returns 400.
4846            if !reasoning_text.is_empty() {
4847                let item = TranscriptItem::ReasoningSummary(ReasoningSummary {
4848                    text: std::mem::take(&mut reasoning_text),
4849                });
4850                self.persist_turn_item(&req.thread_id, &turn_id, &item)
4851                    .await?;
4852                transcript.push(item);
4853            }
4854            for call in &tool_calls {
4855                let tool_item = TranscriptItem::ToolCall(ToolCallRecord {
4856                    id: call.id.clone(),
4857                    name: call.name.clone(),
4858                    arguments: call.arguments.clone(),
4859                });
4860                self.persist_turn_item(&req.thread_id, &turn_id, &tool_item)
4861                    .await?;
4862                transcript.push(tool_item);
4863                self.persist_model_profile_segment(
4864                    &req.thread_id,
4865                    &turn_id,
4866                    model_profile.as_ref(),
4867                    &provider,
4868                    &model,
4869                    "tool_call",
4870                )
4871                .await?;
4872            }
4873            if let Some(deadline) = turn_deadline
4874                && deadline_expired(deadline)
4875            {
4876                self.fail_turn_due_to_deadline(&req.thread_id, &turn_id, deadline, &transcript)
4877                    .await?;
4878                return Ok(TurnRunOutcome::Stopped);
4879            }
4880            let results = self
4881                .route_tool_calls(
4882                    &req.thread_id,
4883                    &turn_id,
4884                    tool_calls,
4885                    parallel_tool_calls,
4886                    Some(workspace.as_str()),
4887                    turn_deadline,
4888                )
4889                .await?;
4890            let reliability_limit = reliability.record_tool_results(
4891                &cfg.reliability,
4892                &results,
4893                runtime_profile == RuntimeProfile::Interactive,
4894            );
4895            for result in results {
4896                verification_gate.record_tool_result(&result);
4897                transcript.push(TranscriptItem::ToolResult(result));
4898                self.persist_model_profile_segment(
4899                    &req.thread_id,
4900                    &turn_id,
4901                    model_profile.as_ref(),
4902                    &provider,
4903                    &model,
4904                    "tool_result",
4905                )
4906                .await?;
4907            }
4908            if let Some(limit) = reliability_limit {
4909                if limit.decision == ReliabilityLimitDecision::RequestContinuation {
4910                    // Loop persistence: rather than ending the turn, reset the
4911                    // consecutive-failure counter, nudge the model to keep going,
4912                    // and continue the round loop. Bounded by the per-turn tool
4913                    // failure ceiling, `max_model_calls_per_turn`, and
4914                    // `MAX_TOOL_ROUNDS_PER_TURN`.
4915                    self.record_reliability_limit_continuation(
4916                        &req.thread_id,
4917                        &turn_id,
4918                        &provider,
4919                        &model,
4920                        limit,
4921                    )
4922                    .await;
4923                    reliability.reset_consecutive_failures();
4924                    let nudge = TranscriptItem::UserMessage(UserMessage::text(
4925                        RELIABILITY_CONTINUATION_PROMPT.to_string(),
4926                    ));
4927                    self.persist_turn_item(&req.thread_id, &turn_id, &nudge)
4928                        .await?;
4929                    transcript.push(nudge);
4930                } else {
4931                    self.fail_turn_due_to_reliability_limit(
4932                        &req.thread_id,
4933                        &turn_id,
4934                        &provider,
4935                        &model,
4936                        limit,
4937                        &transcript,
4938                    )
4939                    .await?;
4940                    return Ok(TurnRunOutcome::Stopped);
4941                }
4942            }
4943            transcript = self
4944                .compact_transcript_if_needed(
4945                    &req.thread_id,
4946                    &turn_id,
4947                    &provider,
4948                    &model,
4949                    transcript,
4950                    self.compaction_options_for_turn(&req.thread_id, !compacted_this_turn),
4951                )
4952                .await?;
4953            compacted_this_turn = compacted_this_turn
4954                || transcript
4955                    .iter()
4956                    .any(crate::compaction::is_compaction_boundary);
4957        }
4958
4959        if exhausted_tool_rounds {
4960            let message =
4961                format!("tool call limit reached after {MAX_TOOL_ROUNDS_PER_TURN} rounds");
4962            self.persist_turn_item(
4963                &req.thread_id,
4964                &turn_id,
4965                &TranscriptItem::Error(ErrorRecord {
4966                    message: message.clone(),
4967                }),
4968            )
4969            .await?;
4970            self.emit(RoderEvent::TurnFailed(TurnFailed {
4971                thread_id: req.thread_id.clone(),
4972                turn_id: turn_id.clone(),
4973                error: message.clone(),
4974                error_kind: None,
4975                usage: None,
4976                timestamp: OffsetDateTime::now_utc(),
4977            }))
4978            .await;
4979            self.complete_team_member_turn_with_result(
4980                &req.thread_id,
4981                &turn_id,
4982                TeamMemberStatus::Failed,
4983                None,
4984                Some(message),
4985            )
4986            .await?;
4987            return Ok(TurnRunOutcome::Stopped);
4988        }
4989
4990        if !final_reasoning_text.is_empty() {
4991            self.persist_turn_item(
4992                &req.thread_id,
4993                &turn_id,
4994                &TranscriptItem::ReasoningSummary(ReasoningSummary {
4995                    text: final_reasoning_text,
4996                }),
4997            )
4998            .await?;
4999        }
5000        for message in final_phase_messages {
5001            self.persist_turn_item(
5002                &req.thread_id,
5003                &turn_id,
5004                &TranscriptItem::AssistantMessage(message),
5005            )
5006            .await?;
5007            self.persist_model_profile_segment(
5008                &req.thread_id,
5009                &turn_id,
5010                model_profile.as_ref(),
5011                &provider,
5012                &model,
5013                "assistant",
5014            )
5015            .await?;
5016        }
5017        if !final_assistant_text.is_empty() {
5018            self.persist_turn_item(
5019                &req.thread_id,
5020                &turn_id,
5021                &TranscriptItem::AssistantMessage(AssistantMessage {
5022                    text: final_assistant_text.clone(),
5023                    phase: Some(FINAL_ANSWER_PHASE.to_string()),
5024                }),
5025            )
5026            .await?;
5027            self.persist_model_profile_segment(
5028                &req.thread_id,
5029                &turn_id,
5030                model_profile.as_ref(),
5031                &provider,
5032                &model,
5033                "assistant",
5034            )
5035            .await?;
5036        }
5037        if let Some(metadata) = final_provider_metadata {
5038            self.persist_turn_item(
5039                &req.thread_id,
5040                &turn_id,
5041                &TranscriptItem::ProviderMetadata(metadata),
5042            )
5043            .await?;
5044        }
5045
5046        let turn_usage_tokens = turn_usage.total_tokens as i64;
5047        let completed_usage = (!turn_usage.is_empty()).then_some(turn_usage.clone());
5048        self.record_thread_usage_metadata(&req.thread_id, &turn_usage)
5049            .await?;
5050        self.goals
5051            .account_turn_usage(
5052                &req.thread_id,
5053                turn_usage_tokens,
5054                OffsetDateTime::now_utc() - turn_started_at,
5055            )
5056            .await?;
5057        let (cleanup, ownership) = self.await_provider_turn_cleanup(&turn_id).await;
5058        self.record_turn_lifecycle_with_ownership(
5059            req.thread_id.clone(),
5060            turn_id.clone(),
5061            TurnLifecycleState::Completed,
5062            cleanup,
5063            None,
5064            ownership,
5065        )
5066        .await;
5067        self.emit(RoderEvent::TurnCompleted(TurnCompleted {
5068            thread_id: req.thread_id.clone(),
5069            turn_id: turn_id.clone(),
5070            usage: completed_usage,
5071            finish_reason: turn_finish_reason,
5072            timestamp: OffsetDateTime::now_utc(),
5073        }))
5074        .await;
5075        self.complete_team_member_turn_with_result(
5076            &req.thread_id,
5077            &turn_id,
5078            TeamMemberStatus::Completed,
5079            (!final_assistant_text.is_empty()).then_some(final_assistant_text),
5080            None,
5081        )
5082        .await?;
5083        Ok(TurnRunOutcome::Completed)
5084    }
5085
5086    async fn drain_turn_steers(&self, turn_id: &TurnId) -> Vec<QueuedTurnSteer> {
5087        let Some(active) = self.active_turns.read().await.get(turn_id).cloned() else {
5088            return Vec::new();
5089        };
5090        let mut steers = active.steers.lock().await;
5091        std::mem::take(&mut *steers)
5092    }
5093
5094    async fn route_tool_calls(
5095        self: &Arc<Self>,
5096        thread_id: &ThreadId,
5097        turn_id: &TurnId,
5098        calls: Vec<ToolCallCompleted>,
5099        parallel: bool,
5100        workspace: Option<&str>,
5101        deadline: Option<OffsetDateTime>,
5102    ) -> anyhow::Result<Vec<ToolResultRecord>> {
5103        // Enforce the agent_swarm exclusivity rule (roadmap 104, Task 2):
5104        // `agent_swarm` must be the only tool call in a model response. A mixed
5105        // or multi-swarm batch is denied wholesale with actionable retry text so
5106        // the model re-issues `agent_swarm` by itself, and every tool_call_id
5107        // still gets a response (keeping the chat-completions transcript valid).
5108        if let Some(violation) = roder_api::subagents::agent_swarm_batch_violation(
5109            calls.iter().map(|call| call.name.as_str()),
5110        ) {
5111            let message = violation.deny_message();
5112            return Ok(calls
5113                .into_iter()
5114                .map(|call| ToolResultRecord {
5115                    id: call.id,
5116                    name: Some(call.name),
5117                    result: message.clone(),
5118                    display_payload: None,
5119                    is_error: true,
5120                })
5121                .collect());
5122        }
5123        // Emit swarm lifecycle events on the bus (roadmap 104, Task 1) so any
5124        // app-server/SDK/TUI client can observe a swarm as a whole; per-child
5125        // progress flows through the existing Subagent* trace events.
5126        let swarm_call = (calls.len() == 1
5127            && calls[0].name == roder_api::subagents::AGENT_SWARM_TOOL_NAME)
5128            .then(|| (calls[0].id.clone(), calls[0].arguments.clone()));
5129        if let Some((tool_id, args)) = &swarm_call {
5130            self.emit(RoderEvent::AgentSwarmStarted(
5131                roder_api::subagents::AgentSwarmStarted {
5132                    thread_id: thread_id.clone(),
5133                    turn_id: turn_id.clone(),
5134                    tool_id: tool_id.clone(),
5135                    child_count: agent_swarm_child_count(args),
5136                    timestamp: OffsetDateTime::now_utc(),
5137                },
5138            ))
5139            .await;
5140        }
5141
5142        let force_sequential = calls
5143            .iter()
5144            .any(|call| crate::agent_control_tools::is_agent_control_tool(&call.name));
5145        let results =
5146            if parallel && !force_sequential {
5147                try_join_all(calls.into_iter().map(|call| {
5148                    self.route_tool_call(thread_id, turn_id, call, workspace, deadline)
5149                }))
5150                .await
5151            } else {
5152                let mut results = Vec::with_capacity(calls.len());
5153                for call in calls {
5154                    results.push(
5155                        self.route_tool_call(thread_id, turn_id, call, workspace, deadline)
5156                            .await?,
5157                    );
5158                }
5159                Ok(results)
5160            }?;
5161
5162        if let Some((tool_id, _)) = &swarm_call
5163            && let Some(result) = results.iter().find(|result| &result.id == tool_id)
5164            && let Some((completed, failed, aborted)) = parse_swarm_counts(&result.result)
5165        {
5166            self.emit(RoderEvent::AgentSwarmCompleted(
5167                roder_api::subagents::AgentSwarmCompleted {
5168                    thread_id: thread_id.clone(),
5169                    turn_id: turn_id.clone(),
5170                    tool_id: tool_id.clone(),
5171                    completed,
5172                    failed,
5173                    aborted,
5174                    timestamp: OffsetDateTime::now_utc(),
5175                },
5176            ))
5177            .await;
5178        }
5179
5180        Ok(results)
5181    }
5182
5183    /// On provider context/prompt-length failures, force-compact (and if needed
5184    /// strip the last bulky item) then retry the current tool round instead of
5185    /// failing the turn. Bounded so a permanently oversized suffix still stops.
5186    async fn try_recover_from_context_limit(
5187        &self,
5188        thread_id: &ThreadId,
5189        turn_id: &TurnId,
5190        provider: &str,
5191        model: &str,
5192        error: &str,
5193        transcript: &mut Vec<TranscriptItem>,
5194        compacted_this_turn: &mut bool,
5195        recovery_attempts: &mut u32,
5196    ) -> anyhow::Result<bool> {
5197        const MAX_CONTEXT_LIMIT_RECOVERY_ATTEMPTS: u32 = 2;
5198        if !crate::compaction::is_context_limit_failure_message(error) {
5199            return Ok(false);
5200        }
5201        if *recovery_attempts >= MAX_CONTEXT_LIMIT_RECOVERY_ATTEMPTS {
5202            return Ok(false);
5203        }
5204        *recovery_attempts = recovery_attempts.saturating_add(1);
5205
5206        let error_item = TranscriptItem::Error(ErrorRecord {
5207            message: error.to_string(),
5208        });
5209        self.persist_turn_item(thread_id, turn_id, &error_item)
5210            .await?;
5211        transcript.push(error_item);
5212
5213        // After the first failed recovery, drop the item ahead of the latest
5214        // user/error so a second compact can succeed when the suffix itself is
5215        // still oversized (e.g. a huge tool result right before the prompt).
5216        if *recovery_attempts > 1 {
5217            *transcript =
5218                crate::compaction::strip_last_message_before_error(std::mem::take(transcript));
5219        }
5220
5221        let force_options = crate::compaction::CompactionOptions {
5222            allow_repeat: true,
5223            force: true,
5224            hysteresis_baseline: None,
5225            preserve_hint: Some(
5226                "Recover from a provider context/prompt-length limit; preserve the active user goal and recent tool outcomes."
5227                    .to_string(),
5228            ),
5229        };
5230        let before_len = transcript.len();
5231        let before_tokens = crate::compaction::estimate_prompt_tokens(transcript);
5232        *transcript = self
5233            .compact_transcript_if_needed(
5234                thread_id,
5235                turn_id,
5236                provider,
5237                model,
5238                std::mem::take(transcript),
5239                force_options,
5240            )
5241            .await?;
5242        let after_tokens = crate::compaction::estimate_prompt_tokens(transcript);
5243        *compacted_this_turn = *compacted_this_turn
5244            || transcript
5245                .iter()
5246                .any(crate::compaction::is_compaction_boundary);
5247
5248        self.emit(RoderEvent::ReliabilityRetryRecorded(
5249            ReliabilityRetryRecorded {
5250                context: ReliabilityContext {
5251                    thread_id: thread_id.clone(),
5252                    turn_id: turn_id.clone(),
5253                    provider: Some(provider.to_string()),
5254                    model: Some(model.to_string()),
5255                    ..ReliabilityContext::default()
5256                },
5257                error_class: ReliabilityErrorClass::ProviderError,
5258                decision: ReliabilityRetryDecision::Retry,
5259                attempt: *recovery_attempts,
5260                max_attempts: MAX_CONTEXT_LIMIT_RECOVERY_ATTEMPTS,
5261                delay_ms: Some(0),
5262                details: ReliabilityDetails::redacted(format!(
5263                    "context_limit_recovery: compact {before_tokens}->{after_tokens} tokens (items {before_len}->{}); {error}",
5264                    transcript.len()
5265                )),
5266                timestamp: OffsetDateTime::now_utc(),
5267            },
5268        ))
5269        .await;
5270
5271        // If force-compact did not shrink anything and we still have budget,
5272        // strip once more so the next round is not identical.
5273        if after_tokens >= before_tokens && *recovery_attempts < MAX_CONTEXT_LIMIT_RECOVERY_ATTEMPTS
5274        {
5275            *transcript =
5276                crate::compaction::strip_last_message_before_error(std::mem::take(transcript));
5277        }
5278        Ok(true)
5279    }
5280
5281    async fn fail_turn_with_error(
5282        &self,
5283        thread_id: &ThreadId,
5284        turn_id: &TurnId,
5285        message: String,
5286    ) -> anyhow::Result<()> {
5287        self.persist_turn_item(
5288            thread_id,
5289            turn_id,
5290            &TranscriptItem::Error(ErrorRecord {
5291                message: message.clone(),
5292            }),
5293        )
5294        .await?;
5295        self.emit(RoderEvent::TurnFailed(TurnFailed {
5296            thread_id: thread_id.clone(),
5297            turn_id: turn_id.clone(),
5298            error: message.clone(),
5299            error_kind: None,
5300            usage: None,
5301            timestamp: OffsetDateTime::now_utc(),
5302        }))
5303        .await;
5304        self.complete_team_member_turn_with_result(
5305            thread_id,
5306            turn_id,
5307            TeamMemberStatus::Failed,
5308            None,
5309            Some(message),
5310        )
5311        .await?;
5312        Ok(())
5313    }
5314
5315    async fn fail_turn_due_to_deadline(
5316        &self,
5317        thread_id: &ThreadId,
5318        turn_id: &TurnId,
5319        deadline: OffsetDateTime,
5320        transcript: &[TranscriptItem],
5321    ) -> anyhow::Result<()> {
5322        let partial_result = turn_partial_result(transcript);
5323        self.emit(RoderEvent::TurnPartialResult(TurnPartialResult {
5324            thread_id: thread_id.clone(),
5325            turn_id: turn_id.clone(),
5326            summary: partial_result.clone(),
5327            timestamp: OffsetDateTime::now_utc(),
5328        }))
5329        .await;
5330        self.emit(RoderEvent::TurnDeadlineExceeded(TurnDeadlineExceeded {
5331            thread_id: thread_id.clone(),
5332            turn_id: turn_id.clone(),
5333            deadline,
5334            partial_result: partial_result.clone(),
5335            timestamp: OffsetDateTime::now_utc(),
5336        }))
5337        .await;
5338        let message = "turn deadline expired".to_string();
5339        self.persist_turn_item(
5340            thread_id,
5341            turn_id,
5342            &TranscriptItem::Error(ErrorRecord {
5343                message: format!("{message}: {partial_result}"),
5344            }),
5345        )
5346        .await?;
5347        self.emit(RoderEvent::TurnFailed(TurnFailed {
5348            thread_id: thread_id.clone(),
5349            turn_id: turn_id.clone(),
5350            error: message.clone(),
5351            error_kind: Some("deadline_timeout".to_string()),
5352            usage: None,
5353            timestamp: OffsetDateTime::now_utc(),
5354        }))
5355        .await;
5356        self.complete_team_member_turn_with_result(
5357            thread_id,
5358            turn_id,
5359            TeamMemberStatus::Failed,
5360            None,
5361            Some(format!("{message}: {partial_result}")),
5362        )
5363        .await?;
5364        Ok(())
5365    }
5366
5367    async fn start_deadline_finalization(
5368        &self,
5369        thread_id: &ThreadId,
5370        turn_id: &TurnId,
5371        transcript: &mut Vec<TranscriptItem>,
5372        remaining_seconds: u64,
5373    ) -> anyhow::Result<()> {
5374        let item = TranscriptItem::UserMessage(crate::deadline_policy::finalization_message(
5375            remaining_seconds,
5376        ));
5377        self.persist_turn_item(thread_id, turn_id, &item).await?;
5378        transcript.push(item);
5379        self.emit(RoderEvent::TurnPartialResult(TurnPartialResult {
5380            thread_id: thread_id.clone(),
5381            turn_id: turn_id.clone(),
5382            summary: turn_partial_result(transcript),
5383            timestamp: OffsetDateTime::now_utc(),
5384        }))
5385        .await;
5386        Ok(())
5387    }
5388
5389    /// Emits the reliability-limit event for a continuation (loop persistence)
5390    /// without failing the turn. The caller resets the failure counter and
5391    /// injects a nudge so the round loop keeps going.
5392    async fn record_reliability_limit_continuation(
5393        &self,
5394        thread_id: &ThreadId,
5395        turn_id: &TurnId,
5396        provider: &str,
5397        model: &str,
5398        limit: ReliabilityLimitHit,
5399    ) {
5400        self.emit(RoderEvent::ReliabilityLimitRecorded(
5401            ReliabilityLimitRecorded {
5402                context: ReliabilityContext {
5403                    thread_id: thread_id.clone(),
5404                    turn_id: turn_id.clone(),
5405                    tool_id: None,
5406                    tool_name: None,
5407                    provider: Some(provider.to_string()),
5408                    model: Some(model.to_string()),
5409                },
5410                error_class: limit.error_class,
5411                limit_kind: limit.limit_kind,
5412                decision: limit.decision,
5413                current: limit.current,
5414                limit: limit.limit,
5415                details: ReliabilityDetails::redacted(&limit.message),
5416                timestamp: OffsetDateTime::now_utc(),
5417            },
5418        ))
5419        .await;
5420    }
5421
5422    async fn fail_turn_due_to_reliability_limit(
5423        &self,
5424        thread_id: &ThreadId,
5425        turn_id: &TurnId,
5426        provider: &str,
5427        model: &str,
5428        limit: ReliabilityLimitHit,
5429        transcript: &[TranscriptItem],
5430    ) -> anyhow::Result<()> {
5431        self.emit(RoderEvent::ReliabilityLimitRecorded(
5432            ReliabilityLimitRecorded {
5433                context: ReliabilityContext {
5434                    thread_id: thread_id.clone(),
5435                    turn_id: turn_id.clone(),
5436                    tool_id: None,
5437                    tool_name: None,
5438                    provider: Some(provider.to_string()),
5439                    model: Some(model.to_string()),
5440                },
5441                error_class: limit.error_class,
5442                limit_kind: limit.limit_kind,
5443                decision: limit.decision,
5444                current: limit.current,
5445                limit: limit.limit,
5446                details: ReliabilityDetails::redacted(&limit.message),
5447                timestamp: OffsetDateTime::now_utc(),
5448            },
5449        ))
5450        .await;
5451        let partial_result = turn_partial_result(transcript);
5452        self.emit(RoderEvent::TurnPartialResult(TurnPartialResult {
5453            thread_id: thread_id.clone(),
5454            turn_id: turn_id.clone(),
5455            summary: partial_result.clone(),
5456            timestamp: OffsetDateTime::now_utc(),
5457        }))
5458        .await;
5459        let message = format!("reliability limit reached: {}", limit.message);
5460        self.persist_turn_item(
5461            thread_id,
5462            turn_id,
5463            &TranscriptItem::Error(ErrorRecord {
5464                message: format!("{message}: {partial_result}"),
5465            }),
5466        )
5467        .await?;
5468        self.emit(RoderEvent::TurnFailed(TurnFailed {
5469            thread_id: thread_id.clone(),
5470            turn_id: turn_id.clone(),
5471            error: message.clone(),
5472            error_kind: Some("reliability_limit".to_string()),
5473            usage: None,
5474            timestamp: OffsetDateTime::now_utc(),
5475        }))
5476        .await;
5477        self.complete_team_member_turn_with_result(
5478            thread_id,
5479            turn_id,
5480            TeamMemberStatus::Failed,
5481            None,
5482            Some(format!("{message}: {partial_result}")),
5483        )
5484        .await?;
5485        Ok(())
5486    }
5487
5488    async fn append_steers(
5489        &self,
5490        req: &StartTurnRequest,
5491        turn_id: &TurnId,
5492        transcript: &mut Vec<TranscriptItem>,
5493        steers: Vec<QueuedTurnSteer>,
5494    ) -> anyhow::Result<()> {
5495        for queued in steers {
5496            let mut steer = queued.message;
5497            steer.text = steer.text.trim().to_string();
5498            if steer.text.is_empty() && steer.images.is_empty() {
5499                continue;
5500            }
5501            let item = TranscriptItem::UserMessage(steer);
5502            self.persist_turn_item(&req.thread_id, turn_id, &item)
5503                .await?;
5504            transcript.push(item);
5505            if let Some(ack) = queued.mailbox_ack {
5506                self.teams
5507                    .mark_mailbox_messages_delivered(&ack.team_id, turn_id, &ack.message_ids)
5508                    .await?;
5509            }
5510        }
5511        Ok(())
5512    }
5513
5514    async fn persist_model_profile_segment(
5515        &self,
5516        thread_id: &ThreadId,
5517        turn_id: &TurnId,
5518        profile: Option<&ModelHarnessProfile>,
5519        provider: &str,
5520        model: &str,
5521        segment: &str,
5522    ) -> anyhow::Result<()> {
5523        let item = TranscriptItem::ProviderMetadata(model_profile_segment_metadata(
5524            profile, provider, model, segment,
5525        ));
5526        self.persist_turn_item(thread_id, turn_id, &item).await
5527    }
5528
5529    /**
5530     * Allowlists apply to built-in tools only; external tools are advertised with their
5531     * host-supplied schemas as given. An external tool shadows a built-in with the same name in
5532     * both advertisement and dispatch (see `route_tool_call`).
5533     */
5534    fn filtered_tool_specs(
5535        &self,
5536        cfg: &RuntimeConfig,
5537        model: &str,
5538        profile: Option<&ModelHarnessProfile>,
5539        thread_allowlist: &[String],
5540        external_tools: &[roder_api::tools::ToolSpec],
5541    ) -> Vec<roder_api::tools::ToolSpec> {
5542        let mut specs = self
5543            .tool_registry
5544            .specs_for_edit_tool_with_schema_policy(
5545                edit_tool_for_model(cfg, model),
5546                schema_policy_for_model(profile),
5547            )
5548            .into_iter()
5549            .filter(|spec| {
5550                allowlist_permits(&cfg.tool_allowlist, &spec.name)
5551                    && allowlist_permits(thread_allowlist, &spec.name)
5552                    && !external_tools.iter().any(|tool| tool.name == spec.name)
5553            })
5554            .collect::<Vec<_>>();
5555        specs.extend(external_tools.iter().cloned());
5556        specs
5557    }
5558
5559    fn task_ledger_tool_specs(
5560        &self,
5561        profile: Option<&ModelHarnessProfile>,
5562    ) -> Vec<roder_api::tools::ToolSpec> {
5563        self.tool_registry
5564            .get(TASK_LEDGER_TOOL_NAME)
5565            .map(|tool| {
5566                tool.spec()
5567                    .normalized_for_model_profile(schema_policy_for_model(profile))
5568            })
5569            .into_iter()
5570            .collect()
5571    }
5572
5573    pub(crate) fn engine_for(&self, provider: &str) -> anyhow::Result<Arc<dyn InferenceEngine>> {
5574        self.registry
5575            .inference_engine(provider)
5576            .or_else(|| {
5577                self.registry
5578                    .default_inference_engine()
5579                    .filter(|engine| provider.is_empty() || engine.id() == provider)
5580            })
5581            .ok_or_else(|| anyhow::anyhow!("inference provider {provider:?} is not registered"))
5582    }
5583
5584    pub async fn emit(&self, event: RoderEvent) -> EventEnvelope {
5585        self.dispatch_local_hook_lifecycle(&event).await;
5586        if let Some(record) = Self::lifecycle_record_for_event(&event) {
5587            let _ = self.persist_turn_lifecycle_record(&record).await;
5588        }
5589        let envelope = self.bus.emit(event);
5590        if let (Some(store), Some(thread_id)) = (&self.thread_store, envelope.thread_id.as_ref())
5591            && should_persist_thread_event(thread_id)
5592        {
5593            let _ = store.append_event(thread_id, &envelope).await;
5594        }
5595        // Registered event sinks (e.g. process extensions) receive the
5596        // persisted envelope through bounded per-sink queues; a slow sink
5597        // never blocks emit or turn progress.
5598        let dispatcher = self
5599            .event_sink_dispatcher
5600            .get_or_init(|| async {
5601                crate::event_sink_dispatch::EventSinkDispatcher::start(
5602                    &self.registry.event_sinks,
5603                    self.bus.clone(),
5604                )
5605            })
5606            .await;
5607        if !dispatcher.is_empty() {
5608            dispatcher.dispatch(&envelope, &self.bus);
5609        }
5610        envelope
5611    }
5612
5613    async fn dispatch_local_hook_lifecycle(&self, event: &RoderEvent) {
5614        // SessionStart belongs to the session, not the turn: firing it from
5615        // TurnStarted re-ran setup hooks on every user message. ThreadCreated
5616        // and ThreadLoaded carry no turn id, so they borrow a synthetic one the
5617        // way the SessionEnd path does.
5618        let session_turn_id = SESSION_HOOK_TURN_ID.to_string();
5619        let (thread_id, turn_id, name, matcher, input) = match event {
5620            RoderEvent::ThreadCreated(event) => (
5621                &event.thread_id,
5622                &session_turn_id,
5623                "SessionStart",
5624                Some("startup"),
5625                serde_json::json!({"source":"startup"}),
5626            ),
5627            RoderEvent::ThreadLoaded(event) => (
5628                &event.thread_id,
5629                &session_turn_id,
5630                "SessionStart",
5631                Some("resume"),
5632                serde_json::json!({"source":"resume"}),
5633            ),
5634            RoderEvent::TurnCompleted(event) => (
5635                &event.thread_id,
5636                &event.turn_id,
5637                "Stop",
5638                None,
5639                serde_json::json!({"finishReason":event.finish_reason}),
5640            ),
5641            RoderEvent::TurnFailed(event) => (
5642                &event.thread_id,
5643                &event.turn_id,
5644                "Stop",
5645                None,
5646                serde_json::json!({"error":event.error}),
5647            ),
5648            RoderEvent::ContextCompactionStarted(event) => (
5649                &event.thread_id,
5650                &event.turn_id,
5651                "PreCompact",
5652                None,
5653                serde_json::json!({"originalItemCount":event.original_item_count,"originalEstimatedTokens":event.original_estimated_tokens}),
5654            ),
5655            RoderEvent::ContextCompactionRecorded(event) => (
5656                &event.thread_id,
5657                &event.turn_id,
5658                "PostCompact",
5659                None,
5660                serde_json::json!({"compactedItemCount":event.compacted_item_count,"compactedEstimatedTokens":event.compacted_estimated_tokens}),
5661            ),
5662            RoderEvent::SubagentStarted(event) => (
5663                &event.parent_thread_id,
5664                &event.parent_turn_id,
5665                "SubagentStart",
5666                Some(event.agent_type.as_str()),
5667                serde_json::json!({"subagentThreadId":event.thread_id,"agentType":event.agent_type,"description":event.description}),
5668            ),
5669            RoderEvent::SubagentCompleted(event) => (
5670                &event.parent_thread_id,
5671                &event.parent_turn_id,
5672                "SubagentStop",
5673                Some(event.agent_type.as_str()),
5674                serde_json::json!({"subagentThreadId":event.thread_id,"agentType":event.agent_type,"exitReason":event.exit_reason}),
5675            ),
5676            RoderEvent::SubagentFailed(event) => (
5677                &event.parent_thread_id,
5678                &event.parent_turn_id,
5679                "SubagentStop",
5680                Some(event.agent_type.as_str()),
5681                serde_json::json!({"subagentThreadId":event.thread_id,"agentType":event.agent_type,"error":event.error}),
5682            ),
5683            _ => return,
5684        };
5685        // One SessionStart per session, whichever event opened it.
5686        if name == "SessionStart" && !self.claim_session_hook_start(thread_id).await {
5687            return;
5688        }
5689        let workspace = self.config.read().await.workspace.clone();
5690        crate::hooks::run_lifecycle(
5691            self,
5692            thread_id,
5693            turn_id,
5694            workspace.as_deref(),
5695            name,
5696            matcher,
5697            input,
5698        )
5699        .await;
5700    }
5701
5702    /// Claim the single `SessionStart` dispatch for a thread, returning whether
5703    /// this caller won it.
5704    pub(crate) async fn claim_session_hook_start(&self, thread_id: &ThreadId) -> bool {
5705        self.session_hook_started
5706            .write()
5707            .await
5708            .insert(thread_id.clone())
5709    }
5710
5711    /// Records a projected thread item event and persists it through the configured thread store.
5712    ///
5713    /// App-server protocol notification bridges currently call this after translating runtime
5714    /// events into item events. Headless runtime consumers that subscribe to `RoderEvent` directly
5715    /// must make the same call if they need the derived item event stream persisted.
5716    pub async fn record_thread_item_event_kind(
5717        &self,
5718        thread_id: &ThreadId,
5719        turn_id: &TurnId,
5720        timestamp: OffsetDateTime,
5721        kind: ThreadItemEventKind,
5722    ) -> anyhow::Result<ThreadItemEvent> {
5723        let seq = self.next_thread_item_event_seq(thread_id).await?;
5724        let item_event = ThreadItemEvent {
5725            seq,
5726            event_id: format!("{turn_id}-item-event-{seq}"),
5727            thread_id: thread_id.clone(),
5728            turn_id: turn_id.clone(),
5729            timestamp,
5730            event: kind,
5731        };
5732        if let Some(store) = &self.thread_store {
5733            store.append_item_event(thread_id, &item_event).await?;
5734        }
5735        self.remember_thread_item_event(&item_event).await?;
5736        Ok(item_event)
5737    }
5738
5739    async fn next_thread_item_event_seq(&self, thread_id: &ThreadId) -> anyhow::Result<u64> {
5740        self.ensure_thread_item_cache(thread_id).await?;
5741        Ok(self
5742            .thread_item_cache
5743            .lock()
5744            .await
5745            .next_item_event_seq(thread_id))
5746    }
5747
5748    pub async fn thread_item_exists(
5749        &self,
5750        thread_id: &ThreadId,
5751        turn_id: &TurnId,
5752        item_id: &str,
5753    ) -> anyhow::Result<bool> {
5754        self.ensure_thread_item_cache(thread_id).await?;
5755        Ok(self
5756            .thread_item_cache
5757            .lock()
5758            .await
5759            .thread_item_exists(thread_id, turn_id, item_id))
5760    }
5761
5762    pub async fn current_reasoning_item_id(
5763        &self,
5764        thread_id: &ThreadId,
5765        turn_id: &TurnId,
5766    ) -> anyhow::Result<Option<String>> {
5767        self.ensure_thread_item_cache(thread_id).await?;
5768        Ok(self
5769            .thread_item_cache
5770            .lock()
5771            .await
5772            .current_reasoning_item_id(thread_id, turn_id))
5773    }
5774
5775    async fn remember_thread_item_event(&self, item_event: &ThreadItemEvent) -> anyhow::Result<()> {
5776        self.ensure_thread_item_cache(&item_event.thread_id).await?;
5777        self.thread_item_cache
5778            .lock()
5779            .await
5780            .remember_item_event(item_event);
5781        Ok(())
5782    }
5783
5784    pub async fn latest_transcript_item_index(
5785        &self,
5786        thread_id: &ThreadId,
5787        turn_id: &TurnId,
5788    ) -> anyhow::Result<Option<usize>> {
5789        self.ensure_thread_item_cache(thread_id).await?;
5790        Ok(self
5791            .thread_item_cache
5792            .lock()
5793            .await
5794            .latest_transcript_item_index(thread_id, turn_id))
5795    }
5796
5797    async fn next_transcript_item_index(
5798        &self,
5799        thread_id: &ThreadId,
5800        turn_id: &TurnId,
5801    ) -> anyhow::Result<usize> {
5802        self.ensure_thread_item_cache(thread_id).await?;
5803        Ok(self
5804            .thread_item_cache
5805            .lock()
5806            .await
5807            .next_transcript_item_index(thread_id, turn_id))
5808    }
5809
5810    async fn remember_transcript_item_index(
5811        &self,
5812        thread_id: &ThreadId,
5813        turn_id: &TurnId,
5814        item_index: usize,
5815    ) -> anyhow::Result<()> {
5816        self.ensure_thread_item_cache(thread_id).await?;
5817        self.thread_item_cache
5818            .lock()
5819            .await
5820            .remember_transcript_item_index(thread_id, turn_id, item_index);
5821        Ok(())
5822    }
5823
5824    async fn ensure_thread_item_cache(&self, thread_id: &ThreadId) -> anyhow::Result<()> {
5825        if self
5826            .thread_item_cache
5827            .lock()
5828            .await
5829            .contains_thread(thread_id)
5830        {
5831            return Ok(());
5832        }
5833
5834        let snapshot = if let Some(store) = &self.thread_store {
5835            store.load_thread(thread_id).await?
5836        } else {
5837            None
5838        };
5839        self.thread_item_cache.lock().await.ensure_thread(
5840            thread_id,
5841            ThreadItemCacheEntry::from_snapshot(snapshot.as_ref()),
5842        );
5843        Ok(())
5844    }
5845
5846    pub(crate) async fn persist_turn_item(
5847        &self,
5848        thread_id: &ThreadId,
5849        turn_id: &TurnId,
5850        item: &TranscriptItem,
5851    ) -> anyhow::Result<()> {
5852        let item_index = self.next_transcript_item_index(thread_id, turn_id).await?;
5853        let timestamp = OffsetDateTime::now_utc();
5854        self.emit(RoderEvent::TranscriptItemAppended(TranscriptItemAppended {
5855            thread_id: thread_id.clone(),
5856            turn_id: turn_id.clone(),
5857            item_type: match item {
5858                TranscriptItem::UserMessage(_) => "user_message",
5859                TranscriptItem::AssistantMessage(_) => "assistant_message",
5860                TranscriptItem::ReasoningSummary(_) => "reasoning_summary",
5861                TranscriptItem::ToolCall(_) => "tool_call",
5862                TranscriptItem::ToolResult(_) => "tool_result",
5863                TranscriptItem::FileChange(_) => "file_change",
5864                TranscriptItem::ContextCompaction(_) => "context_compaction",
5865                TranscriptItem::Error(_) => "error",
5866                TranscriptItem::ProviderMetadata(_) => "provider_metadata",
5867            }
5868            .to_string(),
5869            item_index: Some(item_index),
5870            item: Some(item.clone()),
5871            timestamp,
5872        }))
5873        .await;
5874        self.remember_transcript_item_index(thread_id, turn_id, item_index)
5875            .await?;
5876        Ok(())
5877    }
5878}
5879
5880fn transcript_has_images(transcript: &[TranscriptItem]) -> bool {
5881    transcript.iter().any(|item| {
5882        matches!(
5883            item,
5884            TranscriptItem::UserMessage(message) if !message.images.is_empty()
5885        )
5886    })
5887}
5888
5889fn transcript_has_task_ledger(transcript: &[TranscriptItem]) -> bool {
5890    transcript.iter().any(|item| {
5891        matches!(
5892            item,
5893            TranscriptItem::ToolResult(result)
5894                if result.name.as_deref() == Some(TASK_LEDGER_TOOL_NAME) && !result.is_error
5895        )
5896    })
5897}
5898
5899fn task_ledger_completion_prompt(transcript: &[TranscriptItem]) -> Option<String> {
5900    let latest = transcript.iter().rev().find_map(|item| match item {
5901        TranscriptItem::ToolResult(result)
5902            if result.name.as_deref() == Some(TASK_LEDGER_TOOL_NAME) && !result.is_error =>
5903        {
5904            Some(result.result.as_str())
5905        }
5906        _ => None,
5907    })?;
5908    if !task_ledger_has_open_items(latest) {
5909        return None;
5910    }
5911
5912    let mut ledger = latest.chars().take(1500).collect::<String>();
5913    if latest.chars().nth(1500).is_some() {
5914        ledger.push_str("...");
5915    }
5916    Some(format!(
5917        "Task Ledger Completion Required: the latest task ledger still has pending or in-progress items. Do not provide a final answer yet. Use tools to complete the remaining scoreable work, create or update any required output files, then call `{TASK_LEDGER_TOOL_NAME}` with every task completed and evidence before finalizing.\n\nLatest ledger:\n{ledger}"
5918    ))
5919}
5920
5921fn task_ledger_deadline_completion_prompt(
5922    remaining_seconds: u64,
5923    reserve_seconds: u64,
5924    completion_prompt: &str,
5925) -> String {
5926    format!(
5927        "Eval deadline scoreable completion: {remaining_seconds} seconds remain in the {reserve_seconds}-second finalization reserve. Do not browse, search, or start slow work. Use the available tools now to create or update the required scoreable output files, run only a quick local check if needed, then update the task ledger to completed before finalizing.\n\n{completion_prompt}"
5928    )
5929}
5930
5931fn task_ledger_scoreable_checkpoint_prompt(
5932    remaining_seconds: u64,
5933    completion_prompt: &str,
5934) -> String {
5935    format!(
5936        "Scoreable Output Checkpoint: {remaining_seconds} seconds remain before the eval deadline. Before any further research, browsing, or long commands, use tools now to ensure the required output file(s) exist with the best evidence-backed answer, even if provisional. If a scoreable file already exists, read it and preserve that candidate unless you have stronger task-specific evidence for a replacement. Do not overwrite a plausible dated, historical, or local-evidence candidate with a current live-page, partial-coverage, or weaker guess merely to refresh the checkpoint. You may continue refining afterward, but do not apologize or finalize until the scoreable file exists and the task ledger is updated.\n\n{completion_prompt}"
5937    )
5938}
5939
5940fn task_ledger_has_open_items(ledger: &str) -> bool {
5941    ledger.lines().any(|line| {
5942        let line = line.trim_start();
5943        line.starts_with("- pending:") || line.starts_with("- in_progress:")
5944    })
5945}
5946
5947fn turn_deadline_for_config(cfg: &RuntimeConfig) -> Option<OffsetDateTime> {
5948    cfg.turn_deadline_seconds
5949        .filter(|seconds| *seconds > 0)
5950        .map(|seconds| OffsetDateTime::now_utc() + Duration::seconds(seconds as i64))
5951}
5952
5953fn deadline_expired(deadline: OffsetDateTime) -> bool {
5954    OffsetDateTime::now_utc() >= deadline
5955}
5956
5957pub(crate) fn deadline_remaining_seconds(deadline: Option<OffsetDateTime>) -> Option<u64> {
5958    let deadline = deadline?;
5959    if deadline <= OffsetDateTime::now_utc() {
5960        return Some(0);
5961    }
5962    Some(
5963        (deadline - OffsetDateTime::now_utc())
5964            .unsigned_abs()
5965            .as_secs()
5966            .max(1),
5967    )
5968}
5969
5970fn deadline_instant(deadline: OffsetDateTime) -> tokio::time::Instant {
5971    let now = OffsetDateTime::now_utc();
5972    if deadline <= now {
5973        return tokio::time::Instant::now();
5974    }
5975    tokio::time::Instant::now() + (deadline - now).unsigned_abs()
5976}
5977
5978fn inference_timeout_deadline(
5979    deadline: Option<OffsetDateTime>,
5980    runtime_profile: RuntimeProfile,
5981    task_ledger_required: bool,
5982    reserve_seconds: u64,
5983    finalization_requested: bool,
5984    task_ledger_scoreable_checkpoints: u8,
5985    transcript: &[TranscriptItem],
5986) -> Option<(OffsetDateTime, InferenceTimeoutAction)> {
5987    let deadline = deadline?;
5988    if runtime_profile == RuntimeProfile::Eval
5989        && task_ledger_required
5990        && !finalization_requested
5991        && task_ledger_scoreable_checkpoints < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
5992        && TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS > reserve_seconds
5993        && task_ledger_completion_prompt(transcript).is_some()
5994    {
5995        let checkpoint_deadline =
5996            deadline - Duration::seconds(TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS as i64);
5997        if checkpoint_deadline > OffsetDateTime::now_utc() {
5998            return Some((
5999                checkpoint_deadline,
6000                InferenceTimeoutAction::ScoreableCheckpoint,
6001            ));
6002        }
6003    }
6004    if runtime_profile == RuntimeProfile::Eval && !finalization_requested {
6005        return Some((
6006            deadline - Duration::seconds(reserve_seconds as i64),
6007            InferenceTimeoutAction::Finalization,
6008        ));
6009    }
6010    Some((deadline, InferenceTimeoutAction::Finalization))
6011}
6012
6013fn turn_partial_result(transcript: &[TranscriptItem]) -> String {
6014    let tool_results = transcript
6015        .iter()
6016        .filter(|item| matches!(item, TranscriptItem::ToolResult(_)))
6017        .count();
6018    let assistant_messages = transcript
6019        .iter()
6020        .filter(|item| matches!(item, TranscriptItem::AssistantMessage(_)))
6021        .count();
6022    format!(
6023        "partial turn state: {} transcript items, {assistant_messages} assistant messages, {tool_results} tool results",
6024        transcript.len()
6025    )
6026}
6027
6028fn reasoning_for_model(cfg: &RuntimeConfig, model: &str) -> ReasoningConfig {
6029    let level = effective_reasoning_for_model(cfg, model);
6030    match level.as_str() {
6031        "" | REASONING_NONE => ReasoningConfig::default(),
6032        level => ReasoningConfig {
6033            enabled: true,
6034            level: Some(level.to_string()),
6035        },
6036    }
6037}
6038
6039fn server_side_compaction_threshold(cfg: &RuntimeConfig, model: &str) -> Option<u32> {
6040    let entry = lookup_model(model)?;
6041    if !entry.supports_compaction {
6042        return None;
6043    }
6044    cfg.auto_compact_token_limit
6045        .or_else(|| {
6046            model_profile_for_model(cfg, model).and_then(|profile| profile.auto_compact_token_limit)
6047        })
6048        .or(Some(entry.auto_compact_token_limit))
6049        .filter(|threshold| *threshold > 0)
6050}
6051
6052pub(crate) fn tool_search_for_provider_model(
6053    cfg: &RuntimeConfig,
6054    provider: &str,
6055    model: &str,
6056) -> ToolSearchConfig {
6057    let mut resolved = cfg.tool_search.clone();
6058    if let Some(provider_config) = cfg.provider_tool_search.get(provider) {
6059        provider_config.apply_to(&mut resolved);
6060    }
6061    if let Some(model_config) = cfg.model_tool_search.get(model) {
6062        model_config.apply_to(&mut resolved);
6063    }
6064    resolved
6065}
6066
6067fn parallel_tool_calls_for_model(cfg: &RuntimeConfig, model: &str) -> bool {
6068    cfg.model_parallel_tool_calls
6069        .get(model)
6070        .copied()
6071        .or_else(|| {
6072            model_profile_for_model(cfg, model).and_then(|profile| profile.parallel_tool_calls)
6073        })
6074        .unwrap_or(true)
6075}
6076
6077/// Count the children an `agent_swarm` call will launch from its arguments
6078/// (item-based spawns plus resumes). Lenient: returns 0 on malformed input.
6079fn agent_swarm_child_count(arguments: &str) -> usize {
6080    serde_json::from_str::<roder_api::subagents::AgentSwarmRequest>(arguments)
6081        .map(|request| request.items.len() + request.resume_agent_ids.len())
6082        .unwrap_or(0)
6083}
6084
6085/// Parse `completed`/`failed`/`aborted` counts from an `<agent_swarm_result>`
6086/// summary line, e.g. `<summary>completed: 2, failed: 1</summary>`. Omitted
6087/// buckets default to 0. Returns `None` when the text is not a swarm result.
6088fn parse_swarm_counts(text: &str) -> Option<(usize, usize, usize)> {
6089    if !text.contains("<agent_swarm_result>") {
6090        return None;
6091    }
6092    let bucket = |label: &str| -> usize {
6093        let needle = format!("{label}: ");
6094        text.find(&needle)
6095            .map(|start| start + needle.len())
6096            .map(|start| {
6097                text[start..]
6098                    .chars()
6099                    .take_while(|c| c.is_ascii_digit())
6100                    .collect::<String>()
6101            })
6102            .and_then(|digits| digits.parse().ok())
6103            .unwrap_or(0)
6104    };
6105    Some((bucket("completed"), bucket("failed"), bucket("aborted")))
6106}
6107
6108fn effective_reasoning_for_model(cfg: &RuntimeConfig, model: &str) -> String {
6109    let base_reasoning = default_effective_reasoning_for_model(cfg, model);
6110    if cfg.dynamic_workflows.effort_profile == DynamicWorkflowEffortProfile::Ultracode {
6111        return ultracode_reasoning_level_for_model(
6112            model,
6113            &cfg.speed_policy.ultracode_reasoning,
6114            &base_reasoning,
6115        );
6116    }
6117    base_reasoning
6118}
6119
6120fn default_effective_reasoning_for_model(cfg: &RuntimeConfig, model: &str) -> String {
6121    let Some(entry) = lookup_model(model) else {
6122        return cfg
6123            .reasoning
6124            .clone()
6125            .unwrap_or_else(|| REASONING_NONE.to_string());
6126    };
6127    if entry.supported_reasoning.is_empty() {
6128        return REASONING_NONE.to_string();
6129    }
6130    cfg.reasoning
6131        .as_deref()
6132        .filter(|reasoning| {
6133            entry
6134                .supported_reasoning
6135                .iter()
6136                .any(|option| option.effort == *reasoning)
6137        })
6138        .map(str::to_string)
6139        .or_else(|| {
6140            model_profile_for_model(cfg, model)
6141                .and_then(|profile| profile.reasoning.orientation)
6142                .filter(|reasoning| {
6143                    entry
6144                        .supported_reasoning
6145                        .iter()
6146                        .any(|option| option.effort == reasoning)
6147                })
6148        })
6149        .unwrap_or_else(|| entry.default_reasoning.to_string())
6150}
6151
6152fn validate_reasoning_effort(model: &str, effort: &str) -> anyhow::Result<()> {
6153    if effort == REASONING_NONE && !model_supports_reasoning(model, effort) {
6154        return Ok(());
6155    }
6156    let Some(entry) = lookup_model(model) else {
6157        return Ok(());
6158    };
6159    if entry
6160        .supported_reasoning
6161        .iter()
6162        .any(|option| option.effort == effort)
6163    {
6164        Ok(())
6165    } else {
6166        anyhow::bail!("model {model} does not support reasoning effort {effort}")
6167    }
6168}
6169
6170fn validate_runtime_config_reasoning(cfg: &RuntimeConfig) -> anyhow::Result<()> {
6171    let Some(reasoning) = cfg.reasoning.as_deref() else {
6172        return Ok(());
6173    };
6174    let Some(entry) = lookup_model(&cfg.default_model) else {
6175        return Ok(());
6176    };
6177    if entry.provider != PROVIDER_GEMINI {
6178        return Ok(());
6179    }
6180    validate_reasoning_effort(&cfg.default_model, reasoning)
6181}
6182
6183fn validate_runtime_inference_router_config(
6184    registry: &ExtensionRegistry,
6185    cfg: &RuntimeConfig,
6186) -> anyhow::Result<()> {
6187    if !cfg.inference_router.enabled {
6188        return Ok(());
6189    }
6190    let Some(router_id) = cfg.inference_router.router_id.as_deref() else {
6191        anyhow::bail!("inference_router.enabled requires inference_router.router");
6192    };
6193    if registry.inference_router(router_id).is_some() {
6194        return Ok(());
6195    }
6196    let available = registry
6197        .inference_routers
6198        .iter()
6199        .map(|router| router.id())
6200        .collect::<Vec<_>>()
6201        .join(", ");
6202    if available.is_empty() {
6203        anyhow::bail!("inference router {router_id:?} is not registered");
6204    }
6205    anyhow::bail!(
6206        "inference router {router_id:?} is not registered; available routers: {available}"
6207    );
6208}
6209
6210fn model_supports_reasoning(model: &str, effort: &str) -> bool {
6211    lookup_model(model)
6212        .map(|entry| {
6213            entry
6214                .supported_reasoning
6215                .iter()
6216                .any(|option| option.effort == effort)
6217        })
6218        .unwrap_or(false)
6219}
6220
6221fn is_final_answer_phase(phase: Option<&str>) -> bool {
6222    phase.is_none_or(|phase| phase.is_empty() || phase == FINAL_ANSWER_PHASE)
6223}
6224
6225fn edit_tool_for_model<'a>(cfg: &'a RuntimeConfig, model: &'a str) -> Option<&'a str> {
6226    cfg.model_edit_tools
6227        .get(model)
6228        .map(String::as_str)
6229        .or_else(|| {
6230            cfg.model_profiles
6231                .get(model)
6232                .and_then(|profile| profile.edit_tool.as_deref())
6233        })
6234        .or_else(|| lookup_model(model).and_then(|entry| entry.edit_tool))
6235        .or(Some(EDIT_TOOL_EDIT))
6236}
6237
6238fn model_profile_for_model(cfg: &RuntimeConfig, model: &str) -> Option<ModelHarnessProfile> {
6239    cfg.model_profiles
6240        .get(model)
6241        .cloned()
6242        .or_else(|| built_in_model_profile(model))
6243}
6244
6245pub(crate) fn allowlist_permits(allowlist: &[String], tool_name: &str) -> bool {
6246    allowlist.is_empty() || allowlist.iter().any(|allowed| allowed == tool_name)
6247}
6248
6249/// Provider-aware profile resolution for the active turn.
6250///
6251/// Many model ids are shared across providers (for example Cursor proxies
6252/// `claude-opus-4-8`). Resolving the harness profile by id alone picks the
6253/// first catalog entry, which assigns cross-provider ids the wrong family and
6254/// instruction overlay (e.g. a `cursor/claude-opus-4-8` turn would otherwise
6255/// inherit the Anthropic overlay). Prefer the explicit `(provider, model)`
6256/// catalog entry, keeping user-configured profile overrides as the top
6257/// precedence.
6258fn model_profile_for_provider_model(
6259    cfg: &RuntimeConfig,
6260    provider: &str,
6261    model: &str,
6262) -> Option<ModelHarnessProfile> {
6263    cfg.model_profiles
6264        .get(model)
6265        .cloned()
6266        .or_else(|| built_in_model_profile_for_provider(provider, model))
6267}
6268
6269fn schema_policy_for_model(profile: Option<&ModelHarnessProfile>) -> ModelSchemaPolicy {
6270    profile
6271        .map(|profile| profile.schema_policy)
6272        .unwrap_or_default()
6273}
6274
6275fn model_profile_segment_metadata(
6276    profile: Option<&ModelHarnessProfile>,
6277    provider: &str,
6278    model: &str,
6279    segment: &str,
6280) -> serde_json::Value {
6281    serde_json::json!({
6282        "kind": MODEL_PROFILE_TRACE_KIND,
6283        "segment": segment,
6284        "provider": provider,
6285        "model": model,
6286        "profileModel": profile.map(|profile| profile.model.as_str()).unwrap_or(model),
6287        "providerFamily": profile.map(|profile| profile.provider_family),
6288        "editTool": profile.and_then(|profile| profile.edit_tool.as_deref()),
6289        "schemaPolicy": profile.map(|profile| profile.schema_policy),
6290        "instructionOverlay": profile.map(|profile| profile.instruction_overlay),
6291        "parallelToolCalls": profile.and_then(|profile| profile.parallel_tool_calls),
6292        "autoCompactTokenLimit": profile.and_then(|profile| profile.auto_compact_token_limit),
6293    })
6294}
6295
6296fn model_switch_summary(
6297    transcript: &[TranscriptItem],
6298    profile: Option<&ModelHarnessProfile>,
6299    provider: &str,
6300    model: &str,
6301    tools: &[roder_api::tools::ToolSpec],
6302) -> Option<String> {
6303    let previous = latest_model_profile_segment(transcript)?;
6304    let previous_model = previous
6305        .get("model")
6306        .and_then(serde_json::Value::as_str)
6307        .unwrap_or_default();
6308    let previous_provider = previous
6309        .get("provider")
6310        .and_then(serde_json::Value::as_str)
6311        .unwrap_or_default();
6312    if previous_model == model && previous_provider == provider {
6313        return None;
6314    }
6315
6316    let previous_profile = previous
6317        .get("profileModel")
6318        .and_then(serde_json::Value::as_str)
6319        .unwrap_or(previous_model);
6320    let current_profile = profile
6321        .map(|profile| profile.model.as_str())
6322        .unwrap_or(model);
6323    let previous_edit_tool = previous
6324        .get("editTool")
6325        .and_then(serde_json::Value::as_str)
6326        .unwrap_or("none");
6327    let current_edit_tool = profile
6328        .and_then(|profile| profile.edit_tool.as_deref())
6329        .unwrap_or("none");
6330    let tool_names = tools
6331        .iter()
6332        .map(|tool| tool.name.as_str())
6333        .take(12)
6334        .collect::<Vec<_>>()
6335        .join(", ");
6336    Some(format!(
6337        "{MODEL_SWITCH_SUMMARY_PREFIX} previous profile {previous_provider}/{previous_profile} used edit tool {previous_edit_tool}. Current profile {provider}/{current_profile} uses edit tool {current_edit_tool}. Available tools now: {}.",
6338        if tool_names.is_empty() {
6339            "none"
6340        } else {
6341            &tool_names
6342        }
6343    ))
6344}
6345
6346fn latest_model_profile_segment(transcript: &[TranscriptItem]) -> Option<&serde_json::Value> {
6347    transcript.iter().rev().find_map(|item| {
6348        let TranscriptItem::ProviderMetadata(value) = item else {
6349            return None;
6350        };
6351        (value.get("kind").and_then(serde_json::Value::as_str) == Some(MODEL_PROFILE_TRACE_KIND))
6352            .then_some(value)
6353    })
6354}
6355
6356pub fn validate_edit_tool(value: &str) -> anyhow::Result<()> {
6357    match value.trim() {
6358        EDIT_TOOL_PATCH | EDIT_TOOL_EDIT => Ok(()),
6359        _ => anyhow::bail!(
6360            "unsupported edit_tool {value:?}; allowed values: {EDIT_TOOL_PATCH}, {EDIT_TOOL_EDIT}"
6361        ),
6362    }
6363}
6364
6365fn should_persist_thread_event(thread_id: &str) -> bool {
6366    !is_synthetic_event_thread_id(thread_id)
6367}
6368
6369#[cfg(test)]
6370#[path = "runtime/codex_v2_tests.rs"]
6371mod codex_v2_tests;
6372
6373#[cfg(test)]
6374#[path = "runtime/codex_v2_lifecycle_tests.rs"]
6375mod codex_v2_lifecycle_tests;
6376
6377#[cfg(test)]
6378mod tests {
6379    use super::*;
6380    use futures::stream;
6381    use roder_api::catalog::{
6382        PROVIDER_MOCK, REASONING_HIGH, REASONING_LOW, REASONING_MEDIUM, REASONING_MINIMAL,
6383        REASONING_NONE, REASONING_XHIGH,
6384    };
6385    use roder_api::extension::ExtensionRegistryBuilder;
6386    use roder_api::inference::{
6387        CompletionMetadata, InferenceCapabilities, InferenceEngine, InferenceEventStream,
6388        InferenceProviderContext, InferenceTurnContext, MessageDelta, ModelDescriptor,
6389        ModelInstructionOverlay, ModelProfileReasoning, ModelSchemaPolicy, ProviderFamily,
6390        ReasoningEffortDescriptor,
6391    };
6392    use roder_api::inference_routing::{
6393        InferenceRouter, InferenceRoutingContext, InferenceRoutingDecision, InferenceRoutingOutcome,
6394    };
6395    use roder_api::thread::ThreadStoreFactory;
6396    use roder_api::tools::{ToolContributor, ToolExecutor, ToolSpec};
6397    use roder_ext_jsonl_thread_store::store::JsonlThreadStoreFactory;
6398    use std::sync::Mutex as StdMutex;
6399
6400    fn test_workspace() -> String {
6401        std::env::current_dir().unwrap().display().to_string()
6402    }
6403
6404    #[test]
6405    fn interactive_explicit_turn_deadline_has_bounded_remaining_time() {
6406        let config = RuntimeConfig {
6407            runtime_profile: RuntimeProfile::Interactive,
6408            turn_deadline_seconds: Some(60),
6409            ..RuntimeConfig::default()
6410        };
6411
6412        let deadline = turn_deadline_for_config(&config).expect("configured turn deadline");
6413        let remaining = deadline_remaining_seconds(Some(deadline));
6414
6415        assert!(
6416            matches!(remaining, Some(1..=60)),
6417            "interactive deadline must be positive and no longer than configured: {remaining:?}"
6418        );
6419    }
6420
6421    #[test]
6422    fn interactive_without_turn_deadline_stays_unbounded() {
6423        let config = RuntimeConfig {
6424            runtime_profile: RuntimeProfile::Interactive,
6425            turn_deadline_seconds: None,
6426            ..RuntimeConfig::default()
6427        };
6428
6429        assert_eq!(turn_deadline_for_config(&config), None);
6430    }
6431
6432    struct MetadataMissingStore;
6433
6434    #[async_trait::async_trait]
6435    impl ThreadStore for MetadataMissingStore {
6436        fn id(&self) -> roder_api::thread::ThreadStoreId {
6437            "metadata-missing-store".to_string()
6438        }
6439
6440        async fn create_thread(&self, metadata: ThreadMetadata) -> anyhow::Result<ThreadMetadata> {
6441            Ok(metadata)
6442        }
6443
6444        async fn list_threads(&self) -> anyhow::Result<Vec<ThreadMetadata>> {
6445            Ok(Vec::new())
6446        }
6447
6448        async fn load_thread(
6449            &self,
6450            _thread_id: &ThreadId,
6451        ) -> anyhow::Result<Option<ThreadSnapshot>> {
6452            Ok(Some(ThreadSnapshot {
6453                metadata: None,
6454                ..ThreadSnapshot::default()
6455            }))
6456        }
6457
6458        async fn append_event(
6459            &self,
6460            _thread_id: &ThreadId,
6461            _envelope: &EventEnvelope,
6462        ) -> anyhow::Result<()> {
6463            Ok(())
6464        }
6465    }
6466
6467    struct MetadataMissingStoreFactory;
6468
6469    impl ThreadStoreFactory for MetadataMissingStoreFactory {
6470        fn id(&self) -> roder_api::thread::ThreadStoreId {
6471            "metadata-missing-store".to_string()
6472        }
6473
6474        fn create(&self) -> Arc<dyn ThreadStore> {
6475            Arc::new(MetadataMissingStore)
6476        }
6477    }
6478
6479    #[test]
6480    fn synthetic_app_server_events_are_not_thread_events() {
6481        for thread_id in ["app-server", "runtime", "thread-workflow"] {
6482            assert!(!should_persist_thread_event(thread_id));
6483        }
6484        assert!(should_persist_thread_event("thread-discovery"));
6485        assert!(should_persist_thread_event("thread-plan"));
6486        assert!(should_persist_thread_event("thread-process"));
6487        assert!(should_persist_thread_event("thread-1"));
6488    }
6489
6490    #[test]
6491    fn server_side_compaction_uses_catalog_ninety_percent_default() {
6492        assert_eq!(
6493            server_side_compaction_threshold(&RuntimeConfig::default(), "gpt-5.5"),
6494            Some(945_000)
6495        );
6496        assert_eq!(
6497            server_side_compaction_threshold(&RuntimeConfig::default(), "gpt-5.3-codex-spark"),
6498            Some(115_200)
6499        );
6500    }
6501
6502    #[test]
6503    fn server_side_compaction_respects_explicit_config_override() {
6504        let cfg = RuntimeConfig {
6505            auto_compact_token_limit: Some(123_456),
6506            ..RuntimeConfig::default()
6507        };
6508
6509        assert_eq!(
6510            server_side_compaction_threshold(&cfg, "gpt-5.5"),
6511            Some(123_456)
6512        );
6513    }
6514
6515    #[tokio::test]
6516    async fn pre_request_compaction_runs_when_server_side_model_is_at_context_window() {
6517        let captured = Arc::new(StdMutex::new(None));
6518        let mut builder = ExtensionRegistryBuilder::new();
6519        builder.inference_engine(Arc::new(CapturingEngine {
6520            request: captured.clone(),
6521        }));
6522        let thread_root = std::env::temp_dir().join(format!(
6523            "roder-pre-request-compaction-{}",
6524            uuid::Uuid::new_v4()
6525        ));
6526        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
6527            base_path: thread_root.clone(),
6528        }));
6529        let runtime = Arc::new(
6530            Runtime::new(
6531                builder.build().unwrap(),
6532                RuntimeConfig {
6533                    default_provider: PROVIDER_MOCK.to_string(),
6534                    default_model: "gpt-5.5".to_string(),
6535                    file_backed_dynamic_context: true,
6536                    ..RuntimeConfig::default()
6537                },
6538            )
6539            .unwrap(),
6540        );
6541        let thread_id = runtime
6542            .create_thread(Some("Pre-request compaction".to_string()))
6543            .await
6544            .unwrap()
6545            .thread_id;
6546        let old_turn = "old-turn".to_string();
6547        runtime
6548            .persist_turn_item(
6549                &thread_id,
6550                &old_turn,
6551                &TranscriptItem::UserMessage(UserMessage::text("old context ".repeat(4_300_000))),
6552            )
6553            .await
6554            .unwrap();
6555
6556        let mut events = runtime.subscribe_events();
6557        runtime
6558            .start_turn(StartTurnRequest {
6559                thread_id: thread_id.clone(),
6560                message: "continue".to_string(),
6561                images: Vec::new(),
6562                provider_override: None,
6563                model_override: None,
6564                reasoning_override: None,
6565                workspace: test_workspace(),
6566                instructions: InstructionBundle::default(),
6567                developer_context: None,
6568                task_ledger_required: false,
6569                service_tier_override: None,
6570            })
6571            .await
6572            .unwrap();
6573        loop {
6574            let envelope = tokio::time::timeout(std::time::Duration::from_secs(5), events.recv())
6575                .await
6576                .unwrap()
6577                .unwrap();
6578            if envelope.thread_id.as_deref() == Some(&thread_id)
6579                && matches!(envelope.event, RoderEvent::TurnCompleted(_))
6580            {
6581                break;
6582            }
6583        }
6584
6585        let request = captured.lock().unwrap().clone().unwrap();
6586        assert!(
6587            matches!(
6588                request.transcript.first(),
6589                Some(TranscriptItem::ContextCompaction(_))
6590            ),
6591            "provider request should start with a local emergency compaction item"
6592        );
6593        assert!(
6594            request.transcript.len() < 4,
6595            "provider request should not replay the full oversized prior transcript: {:?}",
6596            request.transcript
6597        );
6598
6599        let _ = std::fs::remove_dir_all(thread_root);
6600    }
6601
6602    #[tokio::test]
6603    async fn continue_after_context_window_failure_compacts_before_provider_request() {
6604        let captured = Arc::new(StdMutex::new(None));
6605        let mut builder = ExtensionRegistryBuilder::new();
6606        builder.inference_engine(Arc::new(CapturingEngine {
6607            request: captured.clone(),
6608        }));
6609        let thread_root = std::env::temp_dir().join(format!(
6610            "roder-context-failure-continue-{}",
6611            uuid::Uuid::new_v4()
6612        ));
6613        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
6614            base_path: thread_root.clone(),
6615        }));
6616        let runtime = Arc::new(
6617            Runtime::new(
6618                builder.build().unwrap(),
6619                RuntimeConfig {
6620                    default_provider: PROVIDER_MOCK.to_string(),
6621                    default_model: "gpt-5.5".to_string(),
6622                    file_backed_dynamic_context: true,
6623                    ..RuntimeConfig::default()
6624                },
6625            )
6626            .unwrap(),
6627        );
6628        let thread_id = runtime
6629            .create_thread(Some("Context failure continue".to_string()))
6630            .await
6631            .unwrap()
6632            .thread_id;
6633        let failed_turn = "failed-turn".to_string();
6634        runtime
6635            .persist_turn_item(
6636                &thread_id,
6637                &failed_turn,
6638                &TranscriptItem::UserMessage(UserMessage::text("old work ".repeat(10_000))),
6639            )
6640            .await
6641            .unwrap();
6642        runtime
6643            .persist_turn_item(
6644                &thread_id,
6645                &failed_turn,
6646                &TranscriptItem::Error(ErrorRecord {
6647                    message: "Your input exceeds the context window of this model. Please adjust your input and try again."
6648                        .to_string(),
6649                }),
6650            )
6651            .await
6652            .unwrap();
6653
6654        let mut events = runtime.subscribe_events();
6655        runtime
6656            .start_turn(StartTurnRequest {
6657                thread_id: thread_id.clone(),
6658                message: "continue".to_string(),
6659                images: Vec::new(),
6660                provider_override: None,
6661                model_override: None,
6662                reasoning_override: None,
6663                workspace: test_workspace(),
6664                instructions: InstructionBundle::default(),
6665                developer_context: None,
6666                task_ledger_required: false,
6667                service_tier_override: None,
6668            })
6669            .await
6670            .unwrap();
6671        loop {
6672            let envelope = tokio::time::timeout(std::time::Duration::from_secs(5), events.recv())
6673                .await
6674                .unwrap()
6675                .unwrap();
6676            if envelope.thread_id.as_deref() == Some(&thread_id)
6677                && matches!(envelope.event, RoderEvent::TurnCompleted(_))
6678            {
6679                break;
6680            }
6681        }
6682
6683        let request = captured.lock().unwrap().clone().unwrap();
6684        assert!(
6685            matches!(
6686                request.transcript.first(),
6687                Some(TranscriptItem::ContextCompaction(_))
6688            ),
6689            "provider request after context-window failure should start with local compaction"
6690        );
6691        assert!(
6692            request
6693                .transcript
6694                .iter()
6695                .any(|item| matches!(item, TranscriptItem::UserMessage(message) if message.text == "continue")),
6696            "current continue prompt must be preserved: {:?}",
6697            request.transcript
6698        );
6699        assert!(
6700            !request.transcript.iter().any(
6701                |item| matches!(item, TranscriptItem::Error(error) if error.message.contains("context window"))
6702            ),
6703            "raw prior context-window error should be summarized, not replayed: {:?}",
6704            request.transcript
6705        );
6706
6707        let _ = std::fs::remove_dir_all(thread_root);
6708    }
6709
6710    #[tokio::test]
6711    async fn workspace_for_thread_falls_back_when_metadata_is_missing() {
6712        let workspace = test_workspace();
6713        let mut builder = ExtensionRegistryBuilder::new();
6714        builder.inference_engine(Arc::new(FakeInferenceEngine));
6715        builder.thread_store_factory(Arc::new(MetadataMissingStoreFactory));
6716        let runtime = Runtime::new(
6717            builder.build().unwrap(),
6718            RuntimeConfig {
6719                workspace: Some(workspace.clone()),
6720                ..RuntimeConfig::default()
6721            },
6722        )
6723        .unwrap();
6724
6725        let resolved = runtime
6726            .workspace_for_thread(&ThreadId::from("thread-workflow"))
6727            .await
6728            .unwrap();
6729
6730        assert_eq!(resolved, workspace);
6731    }
6732
6733    #[tokio::test]
6734    async fn automations_can_create_project_thread_with_model_overrides() {
6735        let runtime = Runtime::fake().unwrap();
6736        let workspace = std::env::temp_dir().join("project");
6737        let metadata = runtime
6738            .create_thread_with(CreateThreadRequest {
6739                title: Some("Automation: nightly status".to_string()),
6740                workspace: workspace.display().to_string(),
6741                workspace_id: None,
6742                root_id: None,
6743                provider: Some("mock".to_string()),
6744                model: Some("mock".to_string()),
6745                selection_mode: None,
6746                tool_allowlist: Vec::new(),
6747                developer_instructions: None,
6748                external_tools: Vec::new(),
6749                runner: None,
6750            })
6751            .await
6752            .unwrap();
6753
6754        assert_eq!(
6755            metadata.title.as_deref(),
6756            Some("Automation: nightly status")
6757        );
6758        assert_eq!(metadata.workspace, workspace.display().to_string());
6759        assert_eq!(metadata.provider.as_deref(), Some("mock"));
6760        assert_eq!(metadata.model.as_deref(), Some("mock"));
6761    }
6762
6763    #[tokio::test]
6764    async fn prior_provider_compaction_boundary_is_used_on_next_turn() {
6765        let captured = Arc::new(StdMutex::new(None));
6766        let mut builder = ExtensionRegistryBuilder::new();
6767        builder.inference_engine(Arc::new(CapturingEngine {
6768            request: captured.clone(),
6769        }));
6770        let thread_root = std::env::temp_dir().join(format!(
6771            "roder-provider-compaction-boundary-{}",
6772            uuid::Uuid::new_v4()
6773        ));
6774        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
6775            base_path: thread_root.clone(),
6776        }));
6777        let runtime = Arc::new(
6778            Runtime::new(
6779                builder.build().unwrap(),
6780                RuntimeConfig {
6781                    default_provider: PROVIDER_MOCK.to_string(),
6782                    default_model: "gpt-5.5".to_string(),
6783                    ..RuntimeConfig::default()
6784                },
6785            )
6786            .unwrap(),
6787        );
6788        let thread_id = runtime
6789            .create_thread(Some("Provider compaction boundary".to_string()))
6790            .await
6791            .unwrap()
6792            .thread_id;
6793        let old_turn = "old-turn".to_string();
6794        runtime
6795            .persist_turn_item(
6796                &thread_id,
6797                &old_turn,
6798                &TranscriptItem::UserMessage(UserMessage::text(
6799                    "old history that must not be replayed after provider compaction",
6800                )),
6801            )
6802            .await
6803            .unwrap();
6804        runtime
6805            .persist_turn_item(
6806                &thread_id,
6807                &old_turn,
6808                &TranscriptItem::AssistantMessage(AssistantMessage {
6809                    text: "old answer".to_string(),
6810                    phase: None,
6811                }),
6812            )
6813            .await
6814            .unwrap();
6815        runtime
6816            .persist_turn_item(
6817                &thread_id,
6818                &old_turn,
6819                &TranscriptItem::ProviderMetadata(serde_json::json!({
6820                    "output": [{
6821                        "id": "cmp_1",
6822                        "type": "compaction",
6823                        "encrypted_content": "opaque-state"
6824                    }]
6825                })),
6826            )
6827            .await
6828            .unwrap();
6829        runtime
6830            .persist_turn_item(
6831                &thread_id,
6832                &old_turn,
6833                &TranscriptItem::AssistantMessage(AssistantMessage {
6834                    text: "after compact".to_string(),
6835                    phase: None,
6836                }),
6837            )
6838            .await
6839            .unwrap();
6840
6841        let mut events = runtime.subscribe_events();
6842        runtime
6843            .start_turn(StartTurnRequest {
6844                thread_id: thread_id.clone(),
6845                message: "continue".to_string(),
6846                images: Vec::new(),
6847                provider_override: None,
6848                model_override: None,
6849                reasoning_override: None,
6850                workspace: test_workspace(),
6851                instructions: InstructionBundle::default(),
6852                developer_context: None,
6853                task_ledger_required: false,
6854                service_tier_override: None,
6855            })
6856            .await
6857            .unwrap();
6858        loop {
6859            let envelope = tokio::time::timeout(std::time::Duration::from_secs(5), events.recv())
6860                .await
6861                .unwrap()
6862                .unwrap();
6863            if envelope.thread_id.as_deref() == Some(&thread_id)
6864                && matches!(envelope.event, RoderEvent::TurnCompleted(_))
6865            {
6866                break;
6867            }
6868        }
6869
6870        let request = captured.lock().unwrap().clone().unwrap();
6871        let compaction_idx = request.transcript.iter().position(|item| {
6872            matches!(
6873                item,
6874                TranscriptItem::ProviderMetadata(metadata)
6875                    if crate::compaction::provider_metadata_has_compaction(metadata)
6876            )
6877        });
6878        assert!(
6879            compaction_idx.is_some(),
6880            "next provider request should include the provider compaction item: {:?}",
6881            request.transcript
6882        );
6883        // Skill/context injectors may prepend non-history items, but no prior-turn
6884        // conversation may appear before the latest provider compaction boundary.
6885        let history_before_boundary =
6886            request
6887                .transcript
6888                .iter()
6889                .take(compaction_idx.unwrap())
6890                .any(|item| match item {
6891                    TranscriptItem::AssistantMessage(_)
6892                    | TranscriptItem::ToolCall(_)
6893                    | TranscriptItem::ToolResult(_) => true,
6894                    TranscriptItem::UserMessage(message) => {
6895                        !message.text.contains("<skills>") && message.text != "continue"
6896                    }
6897                    _ => false,
6898                });
6899        assert!(
6900            !history_before_boundary,
6901            "pre-provider-compaction conversation must not be replayed: {:?}",
6902            request.transcript
6903        );
6904        assert!(
6905            !request.transcript.iter().any(|item| {
6906                matches!(
6907                    item,
6908                    TranscriptItem::UserMessage(message)
6909                        if message.text.contains("old history that must not be replayed")
6910                )
6911            }),
6912            "pre-provider-compaction history must not be replayed: {:?}",
6913            request.transcript
6914        );
6915        assert!(
6916            request.transcript.iter().any(|item| {
6917                matches!(
6918                    item,
6919                    TranscriptItem::UserMessage(message) if message.text == "continue"
6920                )
6921            }),
6922            "current continue prompt must be preserved: {:?}",
6923            request.transcript
6924        );
6925        assert_eq!(
6926            request.runtime.auto_compact_token_limit,
6927            Some(945_000),
6928            "gpt-5.5 should keep server-side auto compaction configured"
6929        );
6930
6931        let _ = std::fs::remove_dir_all(thread_root);
6932    }
6933
6934    #[test]
6935    fn server_side_compaction_is_only_enabled_for_supported_models() {
6936        let cfg = RuntimeConfig {
6937            auto_compact_token_limit: Some(123_456),
6938            ..RuntimeConfig::default()
6939        };
6940
6941        assert_eq!(server_side_compaction_threshold(&cfg, "mock"), None);
6942        assert_eq!(
6943            server_side_compaction_threshold(&cfg, "codex-auto-review"),
6944            None
6945        );
6946    }
6947
6948    #[test]
6949    fn reasoning_is_disabled_for_models_without_reasoning_support() {
6950        let cfg = RuntimeConfig {
6951            reasoning: Some(REASONING_HIGH.to_string()),
6952            ..RuntimeConfig::default()
6953        };
6954
6955        assert_eq!(
6956            effective_reasoning_for_model(&cfg, "claude-haiku-4-5-20251001"),
6957            REASONING_NONE
6958        );
6959        assert_eq!(
6960            reasoning_for_model(&cfg, "claude-haiku-4-5-20251001"),
6961            ReasoningConfig::default()
6962        );
6963    }
6964
6965    #[test]
6966    fn unsupported_configured_reasoning_falls_back_to_model_default() {
6967        let cfg = RuntimeConfig {
6968            reasoning: Some(REASONING_MINIMAL.to_string()),
6969            ..RuntimeConfig::default()
6970        };
6971
6972        assert_eq!(
6973            effective_reasoning_for_model(&cfg, "gpt-5.5"),
6974            REASONING_MEDIUM
6975        );
6976    }
6977
6978    #[test]
6979    fn unsupported_configured_gemini_reasoning_is_rejected() {
6980        let mut builder = ExtensionRegistryBuilder::new();
6981        builder.inference_engine(std::sync::Arc::new(FakeInferenceEngine));
6982
6983        let err = match Runtime::new(
6984            builder.build().unwrap(),
6985            RuntimeConfig {
6986                default_model: "gemini-3.5-flash".to_string(),
6987                reasoning: Some(REASONING_XHIGH.to_string()),
6988                ..RuntimeConfig::default()
6989            },
6990        ) {
6991            Ok(_) => panic!("expected unsupported Gemini reasoning to be rejected"),
6992            Err(err) => err,
6993        };
6994
6995        assert!(
6996            err.to_string()
6997                .contains("model gemini-3.5-flash does not support reasoning effort xhigh")
6998        );
6999    }
7000
7001    #[tokio::test]
7002    async fn selecting_none_for_non_reasoning_model_preserves_stored_preference() {
7003        let runtime = Runtime::new(
7004            Runtime::fake().unwrap().registry,
7005            RuntimeConfig {
7006                reasoning: Some(REASONING_HIGH.to_string()),
7007                ..RuntimeConfig::default()
7008            },
7009        )
7010        .unwrap();
7011
7012        let cfg = runtime
7013            .select_provider(
7014                roder_api::catalog::PROVIDER_MOCK.to_string(),
7015                Some("claude-haiku-4-5-20251001".to_string()),
7016                Some(REASONING_NONE.to_string()),
7017            )
7018            .await
7019            .unwrap();
7020
7021        assert_eq!(cfg.reasoning.as_deref(), Some(REASONING_HIGH));
7022        assert_eq!(runtime.effective_reasoning().await, REASONING_NONE);
7023    }
7024
7025    #[tokio::test]
7026    async fn selecting_none_for_model_that_supports_none_updates_preference() {
7027        let runtime = Runtime::new(
7028            Runtime::fake().unwrap().registry,
7029            RuntimeConfig {
7030                reasoning: Some(REASONING_HIGH.to_string()),
7031                ..RuntimeConfig::default()
7032            },
7033        )
7034        .unwrap();
7035
7036        let cfg = runtime
7037            .select_provider(
7038                roder_api::catalog::PROVIDER_MOCK.to_string(),
7039                Some("mock".to_string()),
7040                Some(REASONING_NONE.to_string()),
7041            )
7042            .await
7043            .unwrap();
7044
7045        assert_eq!(cfg.reasoning.as_deref(), Some(REASONING_NONE));
7046    }
7047
7048    #[test]
7049    fn parallel_tool_calls_default_on_with_model_override() {
7050        assert!(parallel_tool_calls_for_model(
7051            &RuntimeConfig::default(),
7052            "custom-model"
7053        ));
7054
7055        let cfg = RuntimeConfig {
7056            model_parallel_tool_calls: std::collections::HashMap::from([(
7057                "custom-model".to_string(),
7058                false,
7059            )]),
7060            ..RuntimeConfig::default()
7061        };
7062
7063        assert!(!parallel_tool_calls_for_model(&cfg, "custom-model"));
7064        assert!(parallel_tool_calls_for_model(&cfg, "other-model"));
7065    }
7066
7067    #[test]
7068    fn profile_parallel_tool_calls_applies_between_config_and_default() {
7069        let cfg = RuntimeConfig {
7070            model_profiles: std::collections::HashMap::from([(
7071                "gpt-5.5".to_string(),
7072                test_model_profile("gpt-5.5"),
7073            )]),
7074            ..RuntimeConfig::default()
7075        };
7076
7077        assert!(!parallel_tool_calls_for_model(&cfg, "gpt-5.5"));
7078
7079        let cfg = RuntimeConfig {
7080            model_parallel_tool_calls: std::collections::HashMap::from([(
7081                "gpt-5.5".to_string(),
7082                true,
7083            )]),
7084            ..cfg
7085        };
7086
7087        assert!(parallel_tool_calls_for_model(&cfg, "gpt-5.5"));
7088    }
7089
7090    struct CapturingEngine {
7091        request: Arc<StdMutex<Option<AgentInferenceRequest>>>,
7092    }
7093
7094    #[async_trait::async_trait]
7095    impl InferenceEngine for CapturingEngine {
7096        fn id(&self) -> String {
7097            roder_api::catalog::PROVIDER_MOCK.to_string()
7098        }
7099
7100        fn capabilities(&self) -> InferenceCapabilities {
7101            InferenceCapabilities::coding_agent_default()
7102        }
7103
7104        async fn list_models(
7105            &self,
7106            _ctx: InferenceProviderContext<'_>,
7107        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7108            Ok(roder_api::catalog::models_for_provider(
7109                roder_api::catalog::PROVIDER_MOCK,
7110                true,
7111            ))
7112        }
7113
7114        async fn stream_turn(
7115            &self,
7116            _ctx: InferenceTurnContext<'_>,
7117            request: AgentInferenceRequest,
7118        ) -> anyhow::Result<InferenceEventStream> {
7119            *self.request.lock().unwrap() = Some(request);
7120            Ok(Box::pin(stream::iter(vec![
7121                Ok(InferenceEvent::MessageDelta(MessageDelta {
7122                    text: "done".to_string(),
7123                    phase: None,
7124                })),
7125                Ok(InferenceEvent::Completed(CompletionMetadata {
7126                    stop_reason: Some("stop".to_string()),
7127                    provider_response_id: None,
7128                })),
7129            ])))
7130        }
7131    }
7132
7133    struct RoutingCaptureEngine {
7134        id: &'static str,
7135        models: Vec<ModelDescriptor>,
7136        requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
7137    }
7138
7139    #[async_trait::async_trait]
7140    impl InferenceEngine for RoutingCaptureEngine {
7141        fn id(&self) -> String {
7142            self.id.to_string()
7143        }
7144
7145        fn capabilities(&self) -> InferenceCapabilities {
7146            InferenceCapabilities::coding_agent_default()
7147        }
7148
7149        async fn list_models(
7150            &self,
7151            _ctx: InferenceProviderContext<'_>,
7152        ) -> anyhow::Result<Vec<ModelDescriptor>> {
7153            Ok(self.models.clone())
7154        }
7155
7156        async fn stream_turn(
7157            &self,
7158            _ctx: InferenceTurnContext<'_>,
7159            request: AgentInferenceRequest,
7160        ) -> anyhow::Result<InferenceEventStream> {
7161            self.requests.lock().unwrap().push(request);
7162            Ok(Box::pin(stream::iter(vec![
7163                Ok(InferenceEvent::MessageDelta(MessageDelta {
7164                    text: "routed".to_string(),
7165                    phase: None,
7166                })),
7167                Ok(InferenceEvent::Completed(CompletionMetadata {
7168                    stop_reason: Some("stop".to_string()),
7169                    provider_response_id: None,
7170                })),
7171            ])))
7172        }
7173    }
7174
7175    struct StaticRouter {
7176        id: &'static str,
7177        decision: InferenceRoutingDecision,
7178        contexts: Arc<StdMutex<Vec<InferenceRoutingContext>>>,
7179    }
7180
7181    #[async_trait::async_trait]
7182    impl InferenceRouter for StaticRouter {
7183        fn id(&self) -> String {
7184            self.id.to_string()
7185        }
7186
7187        async fn route(
7188            &self,
7189            context: InferenceRoutingContext,
7190        ) -> anyhow::Result<InferenceRoutingDecision> {
7191            self.contexts.lock().unwrap().push(context);
7192            Ok(self.decision.clone())
7193        }
7194    }
7195
7196    fn routing_test_model(id: &str, supported_reasoning: &[&str]) -> ModelDescriptor {
7197        ModelDescriptor {
7198            id: id.to_string(),
7199            name: id.to_string(),
7200            context_window: Some(128_000),
7201            default_reasoning: supported_reasoning
7202                .first()
7203                .map(|effort| (*effort).to_string()),
7204            supported_reasoning: supported_reasoning
7205                .iter()
7206                .map(|effort| ReasoningEffortDescriptor {
7207                    effort: (*effort).to_string(),
7208                    description: format!("{effort} reasoning"),
7209                })
7210                .collect(),
7211        }
7212    }
7213
7214    struct TaskLedgerCompletionGateEngine {
7215        calls: StdMutex<u32>,
7216        requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
7217    }
7218
7219    #[async_trait::async_trait]
7220    impl InferenceEngine for TaskLedgerCompletionGateEngine {
7221        fn id(&self) -> String {
7222            roder_api::catalog::PROVIDER_MOCK.to_string()
7223        }
7224
7225        fn capabilities(&self) -> InferenceCapabilities {
7226            InferenceCapabilities::coding_agent_default()
7227        }
7228
7229        async fn list_models(
7230            &self,
7231            _ctx: InferenceProviderContext<'_>,
7232        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7233            Ok(roder_api::catalog::models_for_provider(
7234                roder_api::catalog::PROVIDER_MOCK,
7235                true,
7236            ))
7237        }
7238
7239        async fn stream_turn(
7240            &self,
7241            _ctx: InferenceTurnContext<'_>,
7242            request: AgentInferenceRequest,
7243        ) -> anyhow::Result<InferenceEventStream> {
7244            self.requests.lock().unwrap().push(request);
7245            let mut calls = self.calls.lock().unwrap();
7246            *calls += 1;
7247            let events = match *calls {
7248                1 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7249                    id: "ledger-open".to_string(),
7250                    name: TASK_LEDGER_TOOL_NAME.to_string(),
7251                    arguments: serde_json::json!({
7252                        "tasks": [
7253                            {
7254                                "id": "inspect",
7255                                "content": "Inspect local assets",
7256                                "status": "completed",
7257                                "evidence": "listed workspace"
7258                            },
7259                            {
7260                                "id": "write",
7261                                "content": "Write /app/result.txt",
7262                                "status": "pending"
7263                            }
7264                        ],
7265                        "requireCompletionEvidence": true
7266                    })
7267                    .to_string(),
7268                }))],
7269                3 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7270                    id: "ledger-complete".to_string(),
7271                    name: TASK_LEDGER_TOOL_NAME.to_string(),
7272                    arguments: serde_json::json!({
7273                        "tasks": [
7274                            {
7275                                "id": "inspect",
7276                                "content": "Inspect local assets",
7277                                "status": "completed",
7278                                "evidence": "listed workspace"
7279                            },
7280                            {
7281                                "id": "write",
7282                                "content": "Write /app/result.txt",
7283                                "status": "completed",
7284                                "evidence": "wrote answer"
7285                            }
7286                        ],
7287                        "requireCompletionEvidence": true
7288                    })
7289                    .to_string(),
7290                }))],
7291                _ => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
7292                    text: "final".to_string(),
7293                    phase: None,
7294                }))],
7295            };
7296            Ok(Box::pin(stream::iter(events.into_iter().chain(
7297                std::iter::once(Ok(InferenceEvent::Completed(CompletionMetadata {
7298                    stop_reason: Some("stop".to_string()),
7299                    provider_response_id: None,
7300                }))),
7301            ))))
7302        }
7303    }
7304
7305    struct VerificationGateEngine {
7306        calls: StdMutex<u32>,
7307    }
7308
7309    #[async_trait::async_trait]
7310    impl InferenceEngine for VerificationGateEngine {
7311        fn id(&self) -> String {
7312            roder_api::catalog::PROVIDER_MOCK.to_string()
7313        }
7314
7315        fn capabilities(&self) -> InferenceCapabilities {
7316            InferenceCapabilities::coding_agent_default()
7317        }
7318
7319        async fn list_models(
7320            &self,
7321            _ctx: InferenceProviderContext<'_>,
7322        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7323            Ok(roder_api::catalog::models_for_provider(
7324                roder_api::catalog::PROVIDER_MOCK,
7325                true,
7326            ))
7327        }
7328
7329        async fn stream_turn(
7330            &self,
7331            _ctx: InferenceTurnContext<'_>,
7332            request: AgentInferenceRequest,
7333        ) -> anyhow::Result<InferenceEventStream> {
7334            let mut calls = self.calls.lock().unwrap();
7335            *calls += 1;
7336            let events = match *calls {
7337                1 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7338                    id: "write-1".to_string(),
7339                    name: "write_file".to_string(),
7340                    arguments: serde_json::json!({
7341                        "path": "src/lib.rs",
7342                        "content": "pub fn answer() -> u8 { 42 }\n"
7343                    })
7344                    .to_string(),
7345                }))],
7346                2 => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
7347                    text: "done too early".to_string(),
7348                    phase: None,
7349                }))],
7350                3 if request.transcript.iter().any(|item| {
7351                    matches!(
7352                        item,
7353                        TranscriptItem::UserMessage(message)
7354                            if message.text.contains("Verification gate blocked final completion")
7355                    )
7356                }) =>
7357                {
7358                    vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7359                        id: "verify-1".to_string(),
7360                        name: crate::verification_gate::VERIFICATION_TOOL_NAME.to_string(),
7361                        arguments: serde_json::json!({
7362                            "originalTask": "write code",
7363                            "changedFiles": ["src/lib.rs"],
7364                            "toolEvidence": ["write_file wrote src/lib.rs"],
7365                            "testsRun": ["cargo test -p roder-core verification_gate"],
7366                            "openGaps": [],
7367                            "status": "completed"
7368                        })
7369                        .to_string(),
7370                    }))]
7371                }
7372                _ => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
7373                    text: "verified final".to_string(),
7374                    phase: None,
7375                }))],
7376            };
7377            Ok(Box::pin(stream::iter(events.into_iter().chain(
7378                std::iter::once(Ok(InferenceEvent::Completed(CompletionMetadata {
7379                    stop_reason: Some("stop".to_string()),
7380                    provider_response_id: None,
7381                }))),
7382            ))))
7383        }
7384    }
7385
7386    struct SpeedPolicyEngine {
7387        calls: StdMutex<u32>,
7388        requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
7389    }
7390
7391    #[async_trait::async_trait]
7392    impl InferenceEngine for SpeedPolicyEngine {
7393        fn id(&self) -> String {
7394            roder_api::catalog::PROVIDER_MOCK.to_string()
7395        }
7396
7397        fn capabilities(&self) -> InferenceCapabilities {
7398            InferenceCapabilities::coding_agent_default()
7399        }
7400
7401        async fn list_models(
7402            &self,
7403            _ctx: InferenceProviderContext<'_>,
7404        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7405            Ok(roder_api::catalog::models_for_provider(
7406                roder_api::catalog::PROVIDER_MOCK,
7407                true,
7408            ))
7409        }
7410
7411        async fn stream_turn(
7412            &self,
7413            _ctx: InferenceTurnContext<'_>,
7414            request: AgentInferenceRequest,
7415        ) -> anyhow::Result<InferenceEventStream> {
7416            self.requests.lock().unwrap().push(request.clone());
7417            let mut calls = self.calls.lock().unwrap();
7418            *calls += 1;
7419            let events = match *calls {
7420                1 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7421                    id: "write-1".to_string(),
7422                    name: "write_file".to_string(),
7423                    arguments: serde_json::json!({
7424                        "path": "src/lib.rs",
7425                        "content": "pub fn answer() -> u8 { 42 }\n"
7426                    })
7427                    .to_string(),
7428                }))],
7429                3 if request.transcript.iter().any(|item| {
7430                    matches!(
7431                        item,
7432                        TranscriptItem::UserMessage(message)
7433                            if message.text.contains("Verification gate blocked final completion")
7434                    )
7435                }) =>
7436                {
7437                    vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7438                        id: "verify-1".to_string(),
7439                        name: crate::verification_gate::VERIFICATION_TOOL_NAME.to_string(),
7440                        arguments: serde_json::json!({
7441                            "originalTask": "write code",
7442                            "changedFiles": ["src/lib.rs"],
7443                            "toolEvidence": ["write_file wrote src/lib.rs"],
7444                            "testsRun": ["cargo test -p roder-core speed_policy"],
7445                            "openGaps": [],
7446                            "status": "completed"
7447                        })
7448                        .to_string(),
7449                    }))]
7450                }
7451                _ => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
7452                    text: "done".to_string(),
7453                    phase: None,
7454                }))],
7455            };
7456            Ok(Box::pin(stream::iter(events.into_iter().chain(
7457                std::iter::once(Ok(InferenceEvent::Completed(CompletionMetadata {
7458                    stop_reason: Some("stop".to_string()),
7459                    provider_response_id: None,
7460                }))),
7461            ))))
7462        }
7463    }
7464
7465    struct SwitchCaptureEngine {
7466        requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
7467    }
7468
7469    #[async_trait::async_trait]
7470    impl InferenceEngine for SwitchCaptureEngine {
7471        fn id(&self) -> String {
7472            roder_api::catalog::PROVIDER_MOCK.to_string()
7473        }
7474
7475        fn capabilities(&self) -> InferenceCapabilities {
7476            InferenceCapabilities::coding_agent_default()
7477        }
7478
7479        async fn list_models(
7480            &self,
7481            _ctx: InferenceProviderContext<'_>,
7482        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7483            Ok(roder_api::catalog::models_for_provider(
7484                roder_api::catalog::PROVIDER_MOCK,
7485                true,
7486            ))
7487        }
7488
7489        async fn stream_turn(
7490            &self,
7491            _ctx: InferenceTurnContext<'_>,
7492            request: AgentInferenceRequest,
7493        ) -> anyhow::Result<InferenceEventStream> {
7494            self.requests.lock().unwrap().push(request);
7495            Ok(Box::pin(stream::iter(vec![
7496                Ok(InferenceEvent::MessageDelta(MessageDelta {
7497                    text: "done".to_string(),
7498                    phase: None,
7499                })),
7500                Ok(InferenceEvent::Completed(CompletionMetadata {
7501                    stop_reason: Some("stop".to_string()),
7502                    provider_response_id: None,
7503                })),
7504            ])))
7505        }
7506    }
7507
7508    struct DeadlineEngine;
7509
7510    #[async_trait::async_trait]
7511    impl InferenceEngine for DeadlineEngine {
7512        fn id(&self) -> String {
7513            roder_api::catalog::PROVIDER_MOCK.to_string()
7514        }
7515
7516        fn capabilities(&self) -> InferenceCapabilities {
7517            InferenceCapabilities::coding_agent_default()
7518        }
7519
7520        async fn list_models(
7521            &self,
7522            _ctx: InferenceProviderContext<'_>,
7523        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7524            Ok(Vec::new())
7525        }
7526
7527        async fn stream_turn(
7528            &self,
7529            _ctx: InferenceTurnContext<'_>,
7530            _request: AgentInferenceRequest,
7531        ) -> anyhow::Result<InferenceEventStream> {
7532            Ok(Box::pin(stream::once(async {
7533                tokio::time::sleep(std::time::Duration::from_secs(60)).await;
7534                Ok(InferenceEvent::MessageDelta(MessageDelta {
7535                    text: "too late".to_string(),
7536                    phase: None,
7537                }))
7538            })))
7539        }
7540    }
7541
7542    struct WriteFileContributor;
7543
7544    impl ToolContributor for WriteFileContributor {
7545        fn id(&self) -> String {
7546            "test-write".to_string()
7547        }
7548
7549        fn contribute(&self, registry: &mut ToolRegistry) -> anyhow::Result<()> {
7550            registry.register(Arc::new(WriteFileTool))
7551        }
7552    }
7553
7554    struct WriteFileTool;
7555
7556    #[async_trait::async_trait]
7557    impl ToolExecutor for WriteFileTool {
7558        fn spec(&self) -> ToolSpec {
7559            ToolSpec {
7560                name: "write_file".to_string(),
7561                description: "Write a test file.".to_string(),
7562                parameters: serde_json::json!({
7563                    "type": "object",
7564                    "properties": {
7565                        "path": { "type": "string" },
7566                        "content": { "type": "string" }
7567                    },
7568                    "required": ["path", "content"],
7569                    "additionalProperties": false
7570                }),
7571            }
7572        }
7573
7574        async fn execute(
7575            &self,
7576            _ctx: ToolExecutionContext,
7577            call: ToolCall,
7578        ) -> anyhow::Result<ToolResult> {
7579            let path = call
7580                .arguments
7581                .get("path")
7582                .and_then(serde_json::Value::as_str)
7583                .unwrap_or("src/lib.rs");
7584            Ok(ToolResult {
7585                id: call.id,
7586                name: call.name,
7587                text: format!("wrote {path}"),
7588                data: serde_json::json!({ "path": path }),
7589                is_error: false,
7590            })
7591        }
7592    }
7593
7594    struct ProfileToolContributor;
7595
7596    impl ToolContributor for ProfileToolContributor {
7597        fn id(&self) -> String {
7598            "profile-tools".to_string()
7599        }
7600
7601        fn contribute(&self, registry: &mut ToolRegistry) -> anyhow::Result<()> {
7602            for name in ["apply_patch", "edit", "multi_edit", "write_file"] {
7603                registry.register(Arc::new(ProfileTool {
7604                    name: name.to_string(),
7605                }))?;
7606            }
7607            Ok(())
7608        }
7609    }
7610
7611    struct ProfileTool {
7612        name: String,
7613    }
7614
7615    #[async_trait::async_trait]
7616    impl ToolExecutor for ProfileTool {
7617        fn spec(&self) -> ToolSpec {
7618            ToolSpec {
7619                name: self.name.clone(),
7620                description: format!("{} test tool", self.name),
7621                parameters: serde_json::json!({
7622                    "type": "object",
7623                    "properties": {
7624                        "path": { "type": "string" },
7625                        "content": { "type": "string" }
7626                    },
7627                    "required": ["path", "content"],
7628                    "additionalProperties": false
7629                }),
7630            }
7631        }
7632
7633        async fn execute(
7634            &self,
7635            _ctx: ToolExecutionContext,
7636            call: ToolCall,
7637        ) -> anyhow::Result<ToolResult> {
7638            Ok(ToolResult {
7639                id: call.id,
7640                name: call.name,
7641                text: "ok".to_string(),
7642                data: serde_json::json!({}),
7643                is_error: false,
7644            })
7645        }
7646    }
7647
7648    fn test_model_profile(model: &str) -> ModelHarnessProfile {
7649        ModelHarnessProfile {
7650            model: model.to_string(),
7651            provider: roder_api::catalog::PROVIDER_OPENAI.to_string(),
7652            provider_family: ProviderFamily::OpenAi,
7653            edit_tool: Some(EDIT_TOOL_EDIT.to_string()),
7654            schema_policy: ModelSchemaPolicy::StandardRequiredFirst,
7655            instruction_overlay: ModelInstructionOverlay::IntuitiveContext,
7656            reasoning: ModelProfileReasoning {
7657                orientation: Some(REASONING_LOW.to_string()),
7658                execution: Some(REASONING_LOW.to_string()),
7659                verification: Some(REASONING_LOW.to_string()),
7660                recovery: Some(REASONING_LOW.to_string()),
7661            },
7662            parallel_tool_calls: Some(false),
7663            auto_compact_token_limit: Some(123_000),
7664        }
7665    }
7666
7667    async fn captured_profile_request(cfg: RuntimeConfig) -> AgentInferenceRequest {
7668        let captured = Arc::new(StdMutex::new(None));
7669        let mut builder = ExtensionRegistryBuilder::new();
7670        builder.inference_engine(Arc::new(CapturingEngine {
7671            request: captured.clone(),
7672        }));
7673        builder.tool_contributor(Arc::new(ProfileToolContributor));
7674        let runtime = Arc::new(Runtime::new(builder.build().unwrap(), cfg).unwrap());
7675        let mut rx = runtime.subscribe_events();
7676        let turn_id = runtime
7677            .start_turn(StartTurnRequest {
7678                thread_id: "thread-model-profile".to_string(),
7679                message: "use profile knobs".to_string(),
7680                images: Vec::new(),
7681                provider_override: None,
7682                model_override: None,
7683                reasoning_override: None,
7684                workspace: test_workspace(),
7685                instructions: InstructionBundle {
7686                    system: None,
7687                    developer: Some("base developer".to_string()),
7688                    developer_context: None,
7689                },
7690                developer_context: None,
7691                task_ledger_required: false,
7692                service_tier_override: None,
7693            })
7694            .await
7695            .unwrap();
7696
7697        tokio::time::timeout(std::time::Duration::from_secs(5), async {
7698            loop {
7699                let envelope = rx.recv().await.unwrap();
7700                if envelope.turn_id.as_deref() != Some(&turn_id) {
7701                    continue;
7702                }
7703                match envelope.event {
7704                    RoderEvent::TurnCompleted(_) => break,
7705                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7706                    _ => {}
7707                }
7708            }
7709        })
7710        .await
7711        .unwrap();
7712
7713        captured.lock().unwrap().clone().unwrap()
7714    }
7715
7716    struct ToolThenStopEngine {
7717        calls: StdMutex<u32>,
7718    }
7719
7720    #[async_trait::async_trait]
7721    impl InferenceEngine for ToolThenStopEngine {
7722        fn id(&self) -> String {
7723            roder_api::catalog::PROVIDER_MOCK.to_string()
7724        }
7725
7726        fn capabilities(&self) -> InferenceCapabilities {
7727            InferenceCapabilities::coding_agent_default()
7728        }
7729
7730        async fn list_models(
7731            &self,
7732            _ctx: InferenceProviderContext<'_>,
7733        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7734            Ok(roder_api::catalog::models_for_provider(
7735                roder_api::catalog::PROVIDER_MOCK,
7736                true,
7737            ))
7738        }
7739
7740        async fn stream_turn(
7741            &self,
7742            _ctx: InferenceTurnContext<'_>,
7743            _request: AgentInferenceRequest,
7744        ) -> anyhow::Result<InferenceEventStream> {
7745            let mut calls = self.calls.lock().unwrap();
7746            *calls += 1;
7747            let events = match *calls {
7748                1 => vec![
7749                    Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7750                        id: "write-1".to_string(),
7751                        name: "write_file".to_string(),
7752                        arguments: serde_json::json!({
7753                            "path": "src/lib.rs",
7754                            "content": "pub fn answer() -> u8 { 42 }\n"
7755                        })
7756                        .to_string(),
7757                    })),
7758                    Ok(InferenceEvent::Completed(CompletionMetadata {
7759                        stop_reason: Some("tool_use".to_string()),
7760                        provider_response_id: None,
7761                    })),
7762                ],
7763                _ => vec![
7764                    Ok(InferenceEvent::MessageDelta(MessageDelta {
7765                        text: "final".to_string(),
7766                        phase: None,
7767                    })),
7768                    Ok(InferenceEvent::Completed(CompletionMetadata {
7769                        stop_reason: Some("end_turn".to_string()),
7770                        provider_response_id: None,
7771                    })),
7772                ],
7773            };
7774            Ok(Box::pin(stream::iter(events)))
7775        }
7776    }
7777
7778    #[tokio::test]
7779    async fn turn_completed_reports_terminal_step_finish_reason() {
7780        let mut builder = ExtensionRegistryBuilder::new();
7781        builder.inference_engine(Arc::new(ToolThenStopEngine {
7782            calls: StdMutex::new(0),
7783        }));
7784        builder.tool_contributor(Arc::new(WriteFileContributor));
7785        let runtime = Arc::new(
7786            Runtime::new(
7787                builder.build().unwrap(),
7788                RuntimeConfig {
7789                    policy_mode: PolicyMode::Bypass,
7790                    agent_swarm_mode: false,
7791                    ultra_mode: false,
7792                    ..RuntimeConfig::default()
7793                },
7794            )
7795            .unwrap(),
7796        );
7797        let mut rx = runtime.subscribe_events();
7798        let turn_id = runtime
7799            .start_turn(StartTurnRequest {
7800                thread_id: "thread-finish-reason".to_string(),
7801                message: "write then finish".to_string(),
7802                images: Vec::new(),
7803                provider_override: None,
7804                model_override: None,
7805                reasoning_override: None,
7806                workspace: test_workspace(),
7807                instructions: InstructionBundle {
7808                    system: None,
7809                    developer: None,
7810                    developer_context: None,
7811                },
7812                developer_context: None,
7813                task_ledger_required: false,
7814                service_tier_override: None,
7815            })
7816            .await
7817            .unwrap();
7818
7819        let completed = tokio::time::timeout(std::time::Duration::from_secs(5), async {
7820            loop {
7821                let envelope = rx.recv().await.unwrap();
7822                if envelope.turn_id.as_deref() != Some(&turn_id) {
7823                    continue;
7824                }
7825                match envelope.event {
7826                    RoderEvent::TurnCompleted(event) => break event,
7827                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7828                    _ => {}
7829                }
7830            }
7831        })
7832        .await
7833        .unwrap();
7834
7835        // The mid-turn tool_use stop reason must not leak; the terminal
7836        // end_turn step decides the turn's finish reason.
7837        assert_eq!(completed.finish_reason.as_deref(), Some("stop"));
7838    }
7839
7840    #[tokio::test]
7841    async fn inference_router_selection_changes_request_model_and_records_event() {
7842        let default_requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7843        let routed_requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7844        let contexts = Arc::new(StdMutex::new(Vec::<InferenceRoutingContext>::new()));
7845        let selected = ModelSelection {
7846            provider: "routed-provider".to_string(),
7847            model: "routed-model".to_string(),
7848        };
7849        let default = ModelSelection {
7850            provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7851            model: "mock".to_string(),
7852        };
7853        let decision = InferenceRoutingDecision {
7854            reasoning: Some(ReasoningConfig {
7855                enabled: true,
7856                level: Some(REASONING_LOW.to_string()),
7857            }),
7858            confidence: Some(0.91),
7859            baseline: Some(default.clone()),
7860            matched_signals: vec![roder_api::inference_routing::InferenceRoutingSignal::new(
7861                "intent", "routine",
7862            )],
7863            ..InferenceRoutingDecision::selected("test-router", selected.clone(), "routine request")
7864        };
7865
7866        let mut builder = ExtensionRegistryBuilder::new();
7867        builder.inference_engine(Arc::new(RoutingCaptureEngine {
7868            id: roder_api::catalog::PROVIDER_MOCK,
7869            models: vec![routing_test_model("mock", &[REASONING_LOW])],
7870            requests: default_requests.clone(),
7871        }));
7872        builder.inference_engine(Arc::new(RoutingCaptureEngine {
7873            id: "routed-provider",
7874            models: vec![routing_test_model(
7875                "routed-model",
7876                &[REASONING_LOW, REASONING_MEDIUM],
7877            )],
7878            requests: routed_requests.clone(),
7879        }));
7880        builder.inference_router(Arc::new(StaticRouter {
7881            id: "test-router",
7882            decision,
7883            contexts: contexts.clone(),
7884        }));
7885        let thread_root =
7886            std::env::temp_dir().join(format!("roder-routing-auto-{}", uuid::Uuid::new_v4()));
7887        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
7888            base_path: thread_root.clone(),
7889        }));
7890        let runtime = Arc::new(
7891            Runtime::new(
7892                builder.build().unwrap(),
7893                RuntimeConfig {
7894                    default_provider: default.provider.clone(),
7895                    default_model: default.model.clone(),
7896                    ..RuntimeConfig::default()
7897                },
7898            )
7899            .unwrap(),
7900        );
7901        let thread_id = runtime
7902            .create_thread_with(CreateThreadRequest {
7903                title: Some("Routing auto".to_string()),
7904                workspace: test_workspace(),
7905                workspace_id: None,
7906                root_id: None,
7907                provider: Some(default.provider.clone()),
7908                model: Some(default.model.clone()),
7909                tool_allowlist: Vec::new(),
7910                developer_instructions: None,
7911                external_tools: Vec::new(),
7912                selection_mode: Some(ModelSelectionMode::auto(
7913                    "test-router:coding",
7914                    "test-router",
7915                    "Auto: Coding",
7916                    default.clone(),
7917                    Some("coding".to_string()),
7918                    None,
7919                )),
7920                runner: None,
7921            })
7922            .await
7923            .unwrap()
7924            .thread_id;
7925        let mut rx = runtime.subscribe_events();
7926        let turn_id = runtime
7927            .start_turn(StartTurnRequest {
7928                thread_id: thread_id.clone(),
7929                message: "small cleanup".to_string(),
7930                images: Vec::new(),
7931                provider_override: None,
7932                model_override: None,
7933                reasoning_override: None,
7934                workspace: test_workspace(),
7935                instructions: InstructionBundle::default(),
7936                developer_context: None,
7937                task_ledger_required: false,
7938                service_tier_override: None,
7939            })
7940            .await
7941            .unwrap();
7942
7943        let mut routing_event = None;
7944        let mut inference_started = None;
7945        tokio::time::timeout(std::time::Duration::from_secs(5), async {
7946            loop {
7947                let envelope = rx.recv().await.unwrap();
7948                if envelope.turn_id.as_deref() != Some(&turn_id) {
7949                    continue;
7950                }
7951                match envelope.event {
7952                    RoderEvent::InferenceRoutingDecision(event) => {
7953                        routing_event = Some(event);
7954                    }
7955                    RoderEvent::InferenceStarted(event) => {
7956                        inference_started = Some(event);
7957                    }
7958                    RoderEvent::TurnCompleted(_) => break,
7959                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7960                    _ => {}
7961                }
7962            }
7963        })
7964        .await
7965        .unwrap();
7966
7967        assert!(default_requests.lock().unwrap().is_empty());
7968        let routed_requests = routed_requests.lock().unwrap();
7969        assert_eq!(routed_requests.len(), 1);
7970        assert_eq!(routed_requests[0].model, selected);
7971        assert_eq!(
7972            routed_requests[0].reasoning.level.as_deref(),
7973            Some(REASONING_LOW)
7974        );
7975        assert_eq!(
7976            routed_requests[0].metadata["inferenceRouting"]["outcome"],
7977            "selected"
7978        );
7979
7980        let routing_event = routing_event.expect("routing decision event");
7981        assert_eq!(routing_event.default_selection, default);
7982        assert_eq!(routing_event.selected_selection, selected);
7983        assert_eq!(
7984            routing_event.decision.outcome,
7985            InferenceRoutingOutcome::Selected
7986        );
7987        assert_eq!(
7988            inference_started.expect("inference started event").model,
7989            selected
7990        );
7991
7992        let contexts = contexts.lock().unwrap();
7993        assert_eq!(contexts.len(), 1);
7994        assert_eq!(contexts[0].default_selection, default);
7995        assert_eq!(contexts[0].candidates.len(), 2);
7996        assert!(
7997            contexts[0]
7998                .signals
7999                .iter()
8000                .any(|signal| signal.key == "profile" && signal.value == "coding")
8001        );
8002        let _ = std::fs::remove_dir_all(thread_root);
8003    }
8004
8005    #[tokio::test]
8006    async fn inference_router_is_bypassed_for_explicit_selection() {
8007        let requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
8008        let contexts = Arc::new(StdMutex::new(Vec::<InferenceRoutingContext>::new()));
8009        let selected = ModelSelection {
8010            provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8011            model: "mock".to_string(),
8012        };
8013
8014        let mut builder = ExtensionRegistryBuilder::new();
8015        builder.inference_engine(Arc::new(RoutingCaptureEngine {
8016            id: roder_api::catalog::PROVIDER_MOCK,
8017            models: vec![routing_test_model("mock", &[REASONING_LOW])],
8018            requests: requests.clone(),
8019        }));
8020        builder.inference_router(Arc::new(StaticRouter {
8021            id: "test-router",
8022            decision: InferenceRoutingDecision::selected(
8023                "test-router",
8024                ModelSelection {
8025                    provider: "missing".to_string(),
8026                    model: "missing".to_string(),
8027                },
8028                "would route if called",
8029            ),
8030            contexts: contexts.clone(),
8031        }));
8032        let thread_root =
8033            std::env::temp_dir().join(format!("roder-routing-explicit-{}", uuid::Uuid::new_v4()));
8034        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8035            base_path: thread_root.clone(),
8036        }));
8037        let runtime = Arc::new(
8038            Runtime::new(
8039                builder.build().unwrap(),
8040                RuntimeConfig {
8041                    default_provider: selected.provider.clone(),
8042                    default_model: selected.model.clone(),
8043                    inference_router: RuntimeInferenceRouterConfig {
8044                        enabled: true,
8045                        router_id: Some("test-router".to_string()),
8046                    },
8047                    ..RuntimeConfig::default()
8048                },
8049            )
8050            .unwrap(),
8051        );
8052        let thread_id = runtime
8053            .create_thread_with(CreateThreadRequest {
8054                title: Some("Routing explicit".to_string()),
8055                workspace: test_workspace(),
8056                workspace_id: None,
8057                root_id: None,
8058                provider: Some(selected.provider.clone()),
8059                model: Some(selected.model.clone()),
8060                tool_allowlist: Vec::new(),
8061                developer_instructions: None,
8062                external_tools: Vec::new(),
8063                selection_mode: Some(ModelSelectionMode::auto(
8064                    "test-router:default",
8065                    "test-router",
8066                    "Auto",
8067                    selected.clone(),
8068                    None,
8069                    None,
8070                )),
8071                runner: None,
8072            })
8073            .await
8074            .unwrap()
8075            .thread_id;
8076        let mut rx = runtime.subscribe_events();
8077        let turn_id = runtime
8078            .start_turn(StartTurnRequest {
8079                thread_id,
8080                message: "use explicit selection".to_string(),
8081                images: Vec::new(),
8082                provider_override: Some(selected.provider.clone()),
8083                model_override: Some(selected.model.clone()),
8084                reasoning_override: None,
8085                workspace: test_workspace(),
8086                instructions: InstructionBundle::default(),
8087                developer_context: None,
8088                task_ledger_required: false,
8089                service_tier_override: None,
8090            })
8091            .await
8092            .unwrap();
8093
8094        let mut saw_routing_event = false;
8095        tokio::time::timeout(std::time::Duration::from_secs(5), async {
8096            loop {
8097                let envelope = rx.recv().await.unwrap();
8098                if envelope.turn_id.as_deref() != Some(&turn_id) {
8099                    continue;
8100                }
8101                match envelope.event {
8102                    RoderEvent::InferenceRoutingDecision(_) => {
8103                        saw_routing_event = true;
8104                    }
8105                    RoderEvent::TurnCompleted(_) => break,
8106                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8107                    _ => {}
8108                }
8109            }
8110        })
8111        .await
8112        .unwrap();
8113
8114        assert!(!saw_routing_event);
8115        assert!(contexts.lock().unwrap().is_empty());
8116        let requests = requests.lock().unwrap();
8117        assert_eq!(requests.len(), 1);
8118        assert_eq!(requests[0].model, selected);
8119        let _ = std::fs::remove_dir_all(thread_root);
8120    }
8121
8122    #[tokio::test]
8123    async fn inference_router_is_bypassed_for_manual_selection_mode() {
8124        let requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
8125        let contexts = Arc::new(StdMutex::new(Vec::<InferenceRoutingContext>::new()));
8126        let selected = ModelSelection {
8127            provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8128            model: "mock".to_string(),
8129        };
8130
8131        let mut builder = ExtensionRegistryBuilder::new();
8132        builder.inference_engine(Arc::new(RoutingCaptureEngine {
8133            id: roder_api::catalog::PROVIDER_MOCK,
8134            models: vec![routing_test_model("mock", &[REASONING_LOW])],
8135            requests: requests.clone(),
8136        }));
8137        builder.inference_router(Arc::new(StaticRouter {
8138            id: "test-router",
8139            decision: InferenceRoutingDecision::selected(
8140                "test-router",
8141                ModelSelection {
8142                    provider: "missing".to_string(),
8143                    model: "missing".to_string(),
8144                },
8145                "would route if called",
8146            ),
8147            contexts: contexts.clone(),
8148        }));
8149        let thread_root =
8150            std::env::temp_dir().join(format!("roder-routing-manual-{}", uuid::Uuid::new_v4()));
8151        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8152            base_path: thread_root.clone(),
8153        }));
8154        let runtime = Arc::new(
8155            Runtime::new(
8156                builder.build().unwrap(),
8157                RuntimeConfig {
8158                    default_provider: selected.provider.clone(),
8159                    default_model: selected.model.clone(),
8160                    inference_router: RuntimeInferenceRouterConfig {
8161                        enabled: true,
8162                        router_id: Some("test-router".to_string()),
8163                    },
8164                    ..RuntimeConfig::default()
8165                },
8166            )
8167            .unwrap(),
8168        );
8169        let thread_id = runtime
8170            .create_thread_with(CreateThreadRequest {
8171                title: Some("Routing manual".to_string()),
8172                workspace: test_workspace(),
8173                workspace_id: None,
8174                root_id: None,
8175                provider: Some(selected.provider.clone()),
8176                model: Some(selected.model.clone()),
8177                tool_allowlist: Vec::new(),
8178                developer_instructions: None,
8179                external_tools: Vec::new(),
8180                selection_mode: Some(ModelSelectionMode::manual(
8181                    selected.provider.clone(),
8182                    selected.model.clone(),
8183                    None,
8184                )),
8185                runner: None,
8186            })
8187            .await
8188            .unwrap()
8189            .thread_id;
8190        let mut rx = runtime.subscribe_events();
8191        let turn_id = runtime
8192            .start_turn(StartTurnRequest {
8193                thread_id,
8194                message: "use selected manual model".to_string(),
8195                images: Vec::new(),
8196                provider_override: None,
8197                model_override: None,
8198                reasoning_override: None,
8199                workspace: test_workspace(),
8200                instructions: InstructionBundle::default(),
8201                developer_context: None,
8202                task_ledger_required: false,
8203                service_tier_override: None,
8204            })
8205            .await
8206            .unwrap();
8207
8208        let mut saw_routing_event = false;
8209        tokio::time::timeout(std::time::Duration::from_secs(5), async {
8210            loop {
8211                let envelope = rx.recv().await.unwrap();
8212                if envelope.turn_id.as_deref() != Some(&turn_id) {
8213                    continue;
8214                }
8215                match envelope.event {
8216                    RoderEvent::InferenceRoutingDecision(_) => {
8217                        saw_routing_event = true;
8218                    }
8219                    RoderEvent::TurnCompleted(_) => break,
8220                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8221                    _ => {}
8222                }
8223            }
8224        })
8225        .await
8226        .unwrap();
8227
8228        assert!(!saw_routing_event);
8229        assert!(contexts.lock().unwrap().is_empty());
8230        let requests = requests.lock().unwrap();
8231        assert_eq!(requests.len(), 1);
8232        assert_eq!(requests[0].model, selected);
8233        let _ = std::fs::remove_dir_all(thread_root);
8234    }
8235
8236    #[test]
8237    fn enabled_inference_router_requires_registered_router() {
8238        let mut builder = ExtensionRegistryBuilder::new();
8239        builder.inference_engine(Arc::new(FakeInferenceEngine));
8240
8241        let err = match Runtime::new(
8242            builder.build().unwrap(),
8243            RuntimeConfig {
8244                inference_router: RuntimeInferenceRouterConfig {
8245                    enabled: true,
8246                    router_id: Some("missing-router".to_string()),
8247                },
8248                ..RuntimeConfig::default()
8249            },
8250        ) {
8251            Ok(_) => panic!("runtime should reject unknown inference router"),
8252            Err(err) => err,
8253        };
8254
8255        assert!(
8256            err.to_string()
8257                .contains("inference router \"missing-router\" is not registered")
8258        );
8259    }
8260
8261    #[tokio::test]
8262    async fn model_profile_routes_request_knobs_to_next_inference() {
8263        let request = captured_profile_request(RuntimeConfig {
8264            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8265            default_model: "gpt-5.5".to_string(),
8266            model_profiles: std::collections::HashMap::from([(
8267                "gpt-5.5".to_string(),
8268                test_model_profile("gpt-5.5"),
8269            )]),
8270            ..RuntimeConfig::default()
8271        })
8272        .await;
8273
8274        let tool_names = request
8275            .tools
8276            .iter()
8277            .map(|tool| tool.name.as_str())
8278            .collect::<Vec<_>>();
8279        assert!(tool_names.contains(&"apply_patch"));
8280        assert!(tool_names.contains(&"edit"));
8281        assert!(tool_names.contains(&"multi_edit"));
8282        assert!(tool_names.contains(&"write_file"));
8283        assert_eq!(request.reasoning.level.as_deref(), Some(REASONING_LOW));
8284        assert_eq!(request.runtime.parallel_tool_calls, Some(false));
8285        assert_eq!(request.runtime.auto_compact_token_limit, Some(123_000));
8286        assert!(
8287            request
8288                .instructions
8289                .developer
8290                .as_deref()
8291                .unwrap_or_default()
8292                .contains("Use the provided context as the current working set")
8293        );
8294        assert_eq!(
8295            request
8296                .metadata
8297                .pointer("/modelProfile/schemaPolicy")
8298                .and_then(serde_json::Value::as_str),
8299            Some("standard_required_first")
8300        );
8301    }
8302
8303    #[tokio::test]
8304    async fn ultra_reasoning_reaches_transport_state_and_enables_proactive_delegation() {
8305        let request = captured_profile_request(RuntimeConfig {
8306            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8307            default_model: "gpt-5.6-sol".to_string(),
8308            reasoning: Some(REASONING_ULTRA.to_string()),
8309            ..RuntimeConfig::default()
8310        })
8311        .await;
8312
8313        assert_eq!(request.reasoning.level.as_deref(), Some(REASONING_ULTRA));
8314        let developer = request
8315            .instructions
8316            .developer
8317            .as_deref()
8318            .expect("ultra developer instructions");
8319        assert!(developer.contains("Proactive multi-agent delegation is active"));
8320    }
8321
8322    #[tokio::test]
8323    async fn lower_sol_effort_requires_explicit_multi_agent_request() {
8324        let request = captured_profile_request(RuntimeConfig {
8325            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8326            default_model: "gpt-5.6-sol".to_string(),
8327            reasoning: Some(roder_api::catalog::REASONING_MEDIUM.to_string()),
8328            ..RuntimeConfig::default()
8329        })
8330        .await;
8331
8332        let developer = request
8333            .instructions
8334            .developer
8335            .as_deref()
8336            .expect("sol developer instructions");
8337        assert!(developer.contains("Do not spawn sub-agents unless"));
8338        assert!(!developer.contains("Proactive multi-agent delegation is active"));
8339    }
8340
8341    #[tokio::test]
8342    async fn luna_does_not_receive_codex_v2_multi_agent_policy() {
8343        let request = captured_profile_request(RuntimeConfig {
8344            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8345            default_model: "gpt-5.6-luna".to_string(),
8346            reasoning: Some(roder_api::catalog::REASONING_MAX.to_string()),
8347            ..RuntimeConfig::default()
8348        })
8349        .await;
8350
8351        let developer = request
8352            .instructions
8353            .developer
8354            .as_deref()
8355            .unwrap_or_default();
8356        assert!(!developer.contains("Do not spawn sub-agents unless"));
8357        assert!(!developer.contains("Proactive multi-agent delegation is active"));
8358    }
8359
8360    #[tokio::test]
8361    async fn ultra_mode_enables_proactive_multi_agent_for_any_model() {
8362        // Luna has no Ultra effort, but ultra mode still injects proactive policy.
8363        let request = captured_profile_request(RuntimeConfig {
8364            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8365            default_model: "gpt-5.6-luna".to_string(),
8366            reasoning: Some(roder_api::catalog::REASONING_MAX.to_string()),
8367            ultra_mode: true,
8368            ..RuntimeConfig::default()
8369        })
8370        .await;
8371
8372        let developer = request
8373            .instructions
8374            .developer
8375            .as_deref()
8376            .expect("ultra mode developer instructions");
8377        assert!(developer.contains("Proactive multi-agent delegation is active"));
8378        assert!(developer.contains("spawn_agent"));
8379    }
8380
8381    #[tokio::test]
8382    async fn ultra_mode_overrides_explicit_request_only_on_sol() {
8383        // Medium Sol effort is normally explicit-request-only; ultra mode flips
8384        // it to proactive without requiring Ultra reasoning effort.
8385        let request = captured_profile_request(RuntimeConfig {
8386            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8387            default_model: "gpt-5.6-sol".to_string(),
8388            reasoning: Some(roder_api::catalog::REASONING_MEDIUM.to_string()),
8389            ultra_mode: true,
8390            ..RuntimeConfig::default()
8391        })
8392        .await;
8393
8394        let developer = request
8395            .instructions
8396            .developer
8397            .as_deref()
8398            .expect("ultra mode sol developer instructions");
8399        assert!(developer.contains("Proactive multi-agent delegation is active"));
8400        assert!(!developer.contains("Do not spawn sub-agents unless"));
8401    }
8402
8403    #[tokio::test]
8404    async fn turn_developer_context_reaches_inference_and_does_not_persist() {
8405        let captured = Arc::new(StdMutex::new(None));
8406        let mut builder = ExtensionRegistryBuilder::new();
8407        builder.inference_engine(Arc::new(CapturingEngine {
8408            request: captured.clone(),
8409        }));
8410        let runtime = Arc::new(
8411            Runtime::new(
8412                builder.build().unwrap(),
8413                RuntimeConfig {
8414                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8415                    default_model: "gpt-5.5".to_string(),
8416                    ..RuntimeConfig::default()
8417                },
8418            )
8419            .unwrap(),
8420        );
8421
8422        async fn run_turn(runtime: &Arc<Runtime>, developer_context: Option<String>) {
8423            let mut rx = runtime.subscribe_events();
8424            let turn_id = runtime
8425                .start_turn(StartTurnRequest {
8426                    thread_id: "thread-turn-context".to_string(),
8427                    message: "hello".to_string(),
8428                    images: Vec::new(),
8429                    provider_override: None,
8430                    model_override: None,
8431                    reasoning_override: None,
8432                    workspace: test_workspace(),
8433                    instructions: InstructionBundle::default(),
8434                    developer_context,
8435                    task_ledger_required: false,
8436                    service_tier_override: None,
8437                })
8438                .await
8439                .unwrap();
8440            tokio::time::timeout(std::time::Duration::from_secs(5), async {
8441                loop {
8442                    let envelope = rx.recv().await.unwrap();
8443                    if envelope.turn_id.as_deref() != Some(&turn_id) {
8444                        continue;
8445                    }
8446                    match envelope.event {
8447                        RoderEvent::TurnCompleted(_) => break,
8448                        RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8449                        _ => {}
8450                    }
8451                }
8452            })
8453            .await
8454            .unwrap();
8455        }
8456
8457        run_turn(
8458            &runtime,
8459            Some("Connected accounts: example-service.".to_string()),
8460        )
8461        .await;
8462        let request = captured.lock().unwrap().clone().unwrap();
8463        assert_eq!(
8464            request.instructions.developer_context.as_deref(),
8465            Some("Connected accounts: example-service.")
8466        );
8467
8468        // The context is per-turn only: the next turn on the same thread
8469        // without a developerContext must not see the previous one.
8470        run_turn(&runtime, None).await;
8471        let request = captured.lock().unwrap().clone().unwrap();
8472        assert_eq!(request.instructions.developer_context, None);
8473    }
8474
8475    #[tokio::test]
8476    async fn tool_search_overrides_route_to_next_inference_request() {
8477        let request = captured_profile_request(RuntimeConfig {
8478            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8479            default_model: "gpt-5.4".to_string(),
8480            tool_search: ToolSearchConfig {
8481                mode: roder_api::inference::ToolSearchMode::Auto,
8482                max_catalog_items: Some(100),
8483                ..ToolSearchConfig::default()
8484            },
8485            provider_tool_search: std::collections::HashMap::from([(
8486                roder_api::catalog::PROVIDER_MOCK.to_string(),
8487                roder_api::inference::ToolSearchConfigOverlay {
8488                    include_skills: Some(false),
8489                    provider_variant: Some(roder_api::inference::ToolSearchProviderVariant::Regex),
8490                    ..Default::default()
8491                },
8492            )]),
8493            model_tool_search: std::collections::HashMap::from([(
8494                "gpt-5.4".to_string(),
8495                roder_api::inference::ToolSearchConfigOverlay {
8496                    mode: Some(roder_api::inference::ToolSearchMode::ProviderNative),
8497                    max_catalog_items: Some(25),
8498                    provider_variant: Some(roder_api::inference::ToolSearchProviderVariant::Bm25),
8499                    ..Default::default()
8500                },
8501            )]),
8502            ..RuntimeConfig::default()
8503        })
8504        .await;
8505
8506        assert_eq!(
8507            request.runtime.tool_search.mode,
8508            roder_api::inference::ToolSearchMode::ProviderNative
8509        );
8510        assert_eq!(request.runtime.tool_search.max_catalog_items, Some(25));
8511        assert_eq!(
8512            request.runtime.tool_search.provider_variant,
8513            roder_api::inference::ToolSearchProviderVariant::Bm25
8514        );
8515    }
8516
8517    #[tokio::test]
8518    async fn context_entrypoint_hints_use_turn_workspace() {
8519        let process_workspace = runtime_test_workspace("entrypoint-process");
8520        let thread_workspace = runtime_test_workspace("entrypoint-thread");
8521        std::fs::create_dir_all(process_workspace.join("src")).unwrap();
8522        std::fs::create_dir_all(thread_workspace.join("src")).unwrap();
8523        std::fs::write(
8524            process_workspace.join("src/sidebar-thread-groups.ts"),
8525            "export const desktopLeak = true;\n",
8526        )
8527        .unwrap();
8528        std::fs::write(
8529            thread_workspace.join("src/voice-plan-feedback.ts"),
8530            "export const voicePlanFeedback = true;\n",
8531        )
8532        .unwrap();
8533
8534        let captured = Arc::new(StdMutex::new(None));
8535        let mut builder = ExtensionRegistryBuilder::new();
8536        builder.inference_engine(Arc::new(CapturingEngine {
8537            request: captured.clone(),
8538        }));
8539        builder.context_planner(Arc::new(roder_context::EntrypointContextPlanner::new(
8540            process_workspace.clone(),
8541        )));
8542        let runtime = Arc::new(
8543            Runtime::new(
8544                builder.build().unwrap(),
8545                RuntimeConfig {
8546                    workspace: Some(process_workspace.display().to_string()),
8547                    ..RuntimeConfig::default()
8548                },
8549            )
8550            .unwrap(),
8551        );
8552        let mut rx = runtime.subscribe_events();
8553
8554        let turn_id = runtime
8555            .start_turn(StartTurnRequest {
8556                thread_id: "thread-workspace-entrypoint".to_string(),
8557                message: "investigate voice plan feedback".to_string(),
8558                images: Vec::new(),
8559                provider_override: None,
8560                model_override: None,
8561                reasoning_override: None,
8562                workspace: thread_workspace.display().to_string(),
8563                instructions: crate::instructions::default_instructions(),
8564                developer_context: None,
8565                task_ledger_required: false,
8566                service_tier_override: None,
8567            })
8568            .await
8569            .unwrap();
8570
8571        tokio::time::timeout(std::time::Duration::from_secs(5), async {
8572            loop {
8573                let envelope = rx.recv().await.unwrap();
8574                if envelope.turn_id.as_deref() != Some(&turn_id) {
8575                    continue;
8576                }
8577                match envelope.event {
8578                    RoderEvent::TurnCompleted(_) => break,
8579                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8580                    _ => {}
8581                }
8582            }
8583        })
8584        .await
8585        .unwrap();
8586
8587        let request = captured.lock().unwrap().clone().expect("captured request");
8588        let transcript_text = request
8589            .transcript
8590            .iter()
8591            .map(|item| match item {
8592                TranscriptItem::UserMessage(message) => message.text.as_str(),
8593                _ => "",
8594            })
8595            .collect::<Vec<_>>()
8596            .join("\n");
8597        assert!(transcript_text.contains("src/voice-plan-feedback.ts"));
8598        assert!(!transcript_text.contains("src/sidebar-thread-groups.ts"));
8599
8600        let _ = std::fs::remove_dir_all(process_workspace);
8601        let _ = std::fs::remove_dir_all(thread_workspace);
8602    }
8603
8604    fn runtime_test_workspace(name: &str) -> std::path::PathBuf {
8605        let path =
8606            std::env::temp_dir().join(format!("roder-runtime-{name}-{}", uuid::Uuid::new_v4()));
8607        let _ = std::fs::remove_dir_all(&path);
8608        std::fs::create_dir_all(&path).unwrap();
8609        path
8610    }
8611
8612    #[tokio::test]
8613    async fn model_profile_user_model_knobs_override_profile_defaults() {
8614        let request = captured_profile_request(RuntimeConfig {
8615            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8616            default_model: "gpt-5.5".to_string(),
8617            reasoning: Some(REASONING_HIGH.to_string()),
8618            auto_compact_token_limit: Some(999),
8619            model_edit_tools: std::collections::HashMap::from([(
8620                "gpt-5.5".to_string(),
8621                EDIT_TOOL_PATCH.to_string(),
8622            )]),
8623            model_parallel_tool_calls: std::collections::HashMap::from([(
8624                "gpt-5.5".to_string(),
8625                true,
8626            )]),
8627            model_profiles: std::collections::HashMap::from([(
8628                "gpt-5.5".to_string(),
8629                test_model_profile("gpt-5.5"),
8630            )]),
8631            ..RuntimeConfig::default()
8632        })
8633        .await;
8634
8635        let tool_names = request
8636            .tools
8637            .iter()
8638            .map(|tool| tool.name.as_str())
8639            .collect::<Vec<_>>();
8640        assert!(tool_names.contains(&"apply_patch"));
8641        assert!(!tool_names.contains(&"edit"));
8642        assert_eq!(request.reasoning.level.as_deref(), Some(REASONING_HIGH));
8643        assert_eq!(request.runtime.parallel_tool_calls, Some(true));
8644        assert_eq!(request.runtime.auto_compact_token_limit, Some(999));
8645    }
8646
8647    #[tokio::test]
8648    async fn runtime_tool_allowlist_filters_advertised_tools() {
8649        let request = captured_profile_request(RuntimeConfig {
8650            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8651            default_model: "gpt-5.5".to_string(),
8652            tool_allowlist: vec!["edit".to_string()],
8653            model_profiles: std::collections::HashMap::from([(
8654                "gpt-5.5".to_string(),
8655                test_model_profile("gpt-5.5"),
8656            )]),
8657            ..RuntimeConfig::default()
8658        })
8659        .await;
8660
8661        let tool_names = request
8662            .tools
8663            .iter()
8664            .map(|tool| tool.name.as_str())
8665            .collect::<Vec<_>>();
8666        assert_eq!(tool_names, vec!["edit"]);
8667    }
8668
8669    /// Builds a store-backed runtime turn for a thread created with the given per-thread
8670    /// overrides and returns the captured inference request.
8671    async fn captured_thread_override_request(
8672        runtime: &Arc<Runtime>,
8673        requests: &Arc<StdMutex<Vec<AgentInferenceRequest>>>,
8674        tool_allowlist: Vec<String>,
8675        developer_instructions: Option<String>,
8676        external_tools: Vec<ToolSpec>,
8677    ) -> AgentInferenceRequest {
8678        let thread_id = runtime
8679            .create_thread_with(CreateThreadRequest {
8680                title: Some("Thread overrides".to_string()),
8681                workspace: test_workspace(),
8682                workspace_id: None,
8683                root_id: None,
8684                provider: None,
8685                model: None,
8686                selection_mode: None,
8687                tool_allowlist,
8688                developer_instructions,
8689                external_tools,
8690                runner: None,
8691            })
8692            .await
8693            .unwrap()
8694            .thread_id;
8695        let mut rx = runtime.subscribe_events();
8696        let turn_id = runtime
8697            .start_turn(StartTurnRequest {
8698                thread_id,
8699                message: "hello".to_string(),
8700                images: Vec::new(),
8701                provider_override: None,
8702                model_override: None,
8703                reasoning_override: None,
8704                workspace: test_workspace(),
8705                instructions: crate::default_instructions(),
8706                developer_context: None,
8707                task_ledger_required: false,
8708                service_tier_override: None,
8709            })
8710            .await
8711            .unwrap();
8712        tokio::time::timeout(std::time::Duration::from_secs(5), async {
8713            loop {
8714                let envelope = rx.recv().await.unwrap();
8715                if envelope.turn_id.as_deref() != Some(&turn_id) {
8716                    continue;
8717                }
8718                match envelope.event {
8719                    RoderEvent::TurnCompleted(_) => break,
8720                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8721                    _ => {}
8722                }
8723            }
8724        })
8725        .await
8726        .unwrap();
8727        requests.lock().unwrap().pop().expect("captured request")
8728    }
8729
8730    #[tokio::test]
8731    async fn thread_tool_allowlist_filters_only_that_thread() {
8732        let requests = Arc::new(StdMutex::new(Vec::new()));
8733        let thread_root =
8734            std::env::temp_dir().join(format!("roder-thread-allowlist-{}", uuid::Uuid::new_v4()));
8735        let mut builder = ExtensionRegistryBuilder::new();
8736        builder.inference_engine(Arc::new(SwitchCaptureEngine {
8737            requests: requests.clone(),
8738        }));
8739        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8740            base_path: thread_root.clone(),
8741        }));
8742        builder.tool_contributor(Arc::new(ProfileToolContributor));
8743        let runtime = Arc::new(
8744            Runtime::new(
8745                builder.build().unwrap(),
8746                RuntimeConfig {
8747                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8748                    default_model: "gpt-5.5".to_string(),
8749                    model_profiles: std::collections::HashMap::from([(
8750                        "gpt-5.5".to_string(),
8751                        test_model_profile("gpt-5.5"),
8752                    )]),
8753                    ..RuntimeConfig::default()
8754                },
8755            )
8756            .unwrap(),
8757        );
8758
8759        let allowlisted = captured_thread_override_request(
8760            &runtime,
8761            &requests,
8762            vec!["edit".to_string()],
8763            None,
8764            Vec::new(),
8765        )
8766        .await;
8767        let unrestricted =
8768            captured_thread_override_request(&runtime, &requests, Vec::new(), None, Vec::new())
8769                .await;
8770
8771        let allowlisted_names = allowlisted
8772            .tools
8773            .iter()
8774            .map(|tool| tool.name.as_str())
8775            .collect::<Vec<_>>();
8776        assert_eq!(allowlisted_names, vec!["edit"]);
8777        let unrestricted_names = unrestricted
8778            .tools
8779            .iter()
8780            .map(|tool| tool.name.as_str())
8781            .collect::<Vec<_>>();
8782        assert!(unrestricted_names.contains(&"edit"));
8783        assert!(unrestricted_names.len() > 1);
8784
8785        let _ = std::fs::remove_dir_all(thread_root);
8786    }
8787
8788    /// Builds a store-backed runtime whose `RuntimeConfig.tool_allowlist` is `["edit"]`.
8789    fn runtime_with_edit_allowlist(
8790        requests: &Arc<StdMutex<Vec<AgentInferenceRequest>>>,
8791        thread_root: &std::path::Path,
8792    ) -> Arc<Runtime> {
8793        let mut builder = ExtensionRegistryBuilder::new();
8794        builder.inference_engine(Arc::new(SwitchCaptureEngine {
8795            requests: requests.clone(),
8796        }));
8797        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8798            base_path: thread_root.to_path_buf(),
8799        }));
8800        builder.tool_contributor(Arc::new(ProfileToolContributor));
8801        Arc::new(
8802            Runtime::new(
8803                builder.build().unwrap(),
8804                RuntimeConfig {
8805                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8806                    default_model: "gpt-5.5".to_string(),
8807                    tool_allowlist: vec!["edit".to_string()],
8808                    model_profiles: std::collections::HashMap::from([(
8809                        "gpt-5.5".to_string(),
8810                        test_model_profile("gpt-5.5"),
8811                    )]),
8812                    ..RuntimeConfig::default()
8813                },
8814            )
8815            .unwrap(),
8816        )
8817    }
8818
8819    #[tokio::test]
8820    async fn runtime_and_thread_allowlists_intersect() {
8821        let requests = Arc::new(StdMutex::new(Vec::new()));
8822        let thread_root = std::env::temp_dir().join(format!(
8823            "roder-allowlist-intersect-{}",
8824            uuid::Uuid::new_v4()
8825        ));
8826        let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8827
8828        let request = captured_thread_override_request(
8829            &runtime,
8830            &requests,
8831            vec!["edit".to_string(), "write_file".to_string()],
8832            None,
8833            Vec::new(),
8834        )
8835        .await;
8836
8837        // The thread allowlist must not re-enable tools the runtime allowlist bans.
8838        let names = request
8839            .tools
8840            .iter()
8841            .map(|tool| tool.name.as_str())
8842            .collect::<Vec<_>>();
8843        assert_eq!(names, vec!["edit"]);
8844
8845        let _ = std::fs::remove_dir_all(thread_root);
8846    }
8847
8848    #[tokio::test]
8849    async fn route_tool_call_denies_tools_outside_allowlists() {
8850        let requests = Arc::new(StdMutex::new(Vec::new()));
8851        let thread_root =
8852            std::env::temp_dir().join(format!("roder-allowlist-dispatch-{}", uuid::Uuid::new_v4()));
8853        let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8854        let thread_id = runtime
8855            .create_thread_with(CreateThreadRequest {
8856                title: Some("Dispatch allowlist".to_string()),
8857                workspace: test_workspace(),
8858                workspace_id: None,
8859                root_id: None,
8860                provider: None,
8861                model: None,
8862                selection_mode: None,
8863                tool_allowlist: vec!["edit".to_string(), "write_file".to_string()],
8864                developer_instructions: None,
8865                external_tools: Vec::new(),
8866                runner: None,
8867            })
8868            .await
8869            .unwrap()
8870            .thread_id;
8871
8872        // write_file is registered and on the thread allowlist but banned by the runtime allowlist.
8873        let result = runtime
8874            .route_tool_call(
8875                &thread_id,
8876                &"turn-allowlist-dispatch".to_string(),
8877                roder_api::inference::ToolCallCompleted {
8878                    id: "call-1".to_string(),
8879                    name: "write_file".to_string(),
8880                    arguments: r#"{"path":"a.txt","content":"hi"}"#.to_string(),
8881                },
8882                None,
8883                None,
8884            )
8885            .await
8886            .unwrap();
8887
8888        assert!(result.is_error);
8889        assert!(
8890            result
8891                .result
8892                .contains("not permitted by the tool allowlist"),
8893            "unexpected result: {}",
8894            result.result
8895        );
8896
8897        let _ = std::fs::remove_dir_all(thread_root);
8898    }
8899
8900    #[tokio::test]
8901    async fn route_tool_calls_denies_agent_swarm_mixed_with_other_tools() {
8902        let requests = Arc::new(StdMutex::new(Vec::new()));
8903        let thread_root =
8904            std::env::temp_dir().join(format!("roder-swarm-exclusive-{}", uuid::Uuid::new_v4()));
8905        let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8906
8907        // A response that mixes agent_swarm with another tool is denied wholesale:
8908        // both calls get an error result with retry guidance, and no tool runs.
8909        let results = runtime
8910            .route_tool_calls(
8911                &"thread-swarm".to_string(),
8912                &"turn-swarm".to_string(),
8913                vec![
8914                    roder_api::inference::ToolCallCompleted {
8915                        id: "swarm-1".to_string(),
8916                        name: "agent_swarm".to_string(),
8917                        arguments: "{}".to_string(),
8918                    },
8919                    roder_api::inference::ToolCallCompleted {
8920                        id: "read-1".to_string(),
8921                        name: "read_file".to_string(),
8922                        arguments: "{}".to_string(),
8923                    },
8924                ],
8925                true,
8926                None,
8927                None,
8928            )
8929            .await
8930            .unwrap();
8931
8932        assert_eq!(results.len(), 2, "every tool_call_id must get a response");
8933        assert_eq!(results[0].id, "swarm-1");
8934        assert_eq!(results[1].id, "read-1");
8935        for result in &results {
8936            assert!(result.is_error);
8937            assert!(
8938                result.result.contains("only tool call"),
8939                "unexpected result: {}",
8940                result.result
8941            );
8942        }
8943
8944        let _ = std::fs::remove_dir_all(thread_root);
8945    }
8946
8947    #[tokio::test]
8948    async fn route_tool_calls_emits_agent_swarm_started_event() {
8949        let requests = Arc::new(StdMutex::new(Vec::new()));
8950        let thread_root =
8951            std::env::temp_dir().join(format!("roder-swarm-started-{}", uuid::Uuid::new_v4()));
8952        let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8953        let mut events = runtime.subscribe_events();
8954
8955        // A single agent_swarm call (the valid shape) emits AgentSwarmStarted on
8956        // the event bus before dispatch, with the child count parsed from args.
8957        let _ = runtime
8958            .route_tool_calls(
8959                &"thread-swarm".to_string(),
8960                &"turn-swarm".to_string(),
8961                vec![roder_api::inference::ToolCallCompleted {
8962                    id: "swarm-1".to_string(),
8963                    name: "agent_swarm".to_string(),
8964                    arguments: r#"{"description":"x","prompt_template":"Read {{item}}","items":["a.rs","b.rs"]}"#
8965                        .to_string(),
8966                }],
8967                true,
8968                None,
8969                None,
8970            )
8971            .await
8972            .unwrap();
8973
8974        let mut started_child_count = None;
8975        for _ in 0..16 {
8976            let envelope = tokio::time::timeout(std::time::Duration::from_secs(2), events.recv())
8977                .await
8978                .unwrap()
8979                .unwrap();
8980            if let RoderEvent::AgentSwarmStarted(event) = envelope.event {
8981                started_child_count = Some(event.child_count);
8982                assert_eq!(event.tool_id, "swarm-1");
8983                break;
8984            }
8985        }
8986        assert_eq!(started_child_count, Some(2));
8987
8988        let _ = std::fs::remove_dir_all(thread_root);
8989    }
8990
8991    #[test]
8992    fn agent_swarm_child_count_sums_items_and_resumes() {
8993        assert_eq!(
8994            agent_swarm_child_count(r#"{"description":"x","items":["a","b","c"]}"#),
8995            3
8996        );
8997        assert_eq!(
8998            agent_swarm_child_count(
8999                r#"{"description":"x","items":["a"],"resume_agent_ids":{"id1":"continue"}}"#
9000            ),
9001            2
9002        );
9003        // Malformed input is lenient.
9004        assert_eq!(agent_swarm_child_count("not json"), 0);
9005    }
9006
9007    #[test]
9008    fn parse_swarm_counts_reads_summary_with_omitted_buckets() {
9009        let text = "<agent_swarm_result>\n<summary>completed: 2, failed: 1</summary>\n</agent_swarm_result>";
9010        assert_eq!(parse_swarm_counts(text), Some((2, 1, 0)));
9011        let text = "<agent_swarm_result>\n<summary>completed: 0</summary>\n</agent_swarm_result>";
9012        assert_eq!(parse_swarm_counts(text), Some((0, 0, 0)));
9013        // Not a swarm result.
9014        assert_eq!(parse_swarm_counts("just text"), None);
9015    }
9016
9017    /// Signals when inference starts, then waits for `proceed` and fails the stream.
9018    struct SignalledFailureEngine {
9019        started: tokio::sync::mpsc::UnboundedSender<()>,
9020        proceed: Arc<tokio::sync::Notify>,
9021    }
9022
9023    #[async_trait::async_trait]
9024    impl InferenceEngine for SignalledFailureEngine {
9025        fn id(&self) -> String {
9026            roder_api::catalog::PROVIDER_MOCK.to_string()
9027        }
9028
9029        fn capabilities(&self) -> InferenceCapabilities {
9030            InferenceCapabilities::coding_agent_default()
9031        }
9032
9033        async fn list_models(
9034            &self,
9035            _ctx: InferenceProviderContext<'_>,
9036        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
9037            Ok(roder_api::catalog::models_for_provider(
9038                roder_api::catalog::PROVIDER_MOCK,
9039                true,
9040            ))
9041        }
9042
9043        async fn stream_turn(
9044            &self,
9045            _ctx: InferenceTurnContext<'_>,
9046            _request: AgentInferenceRequest,
9047        ) -> anyhow::Result<InferenceEventStream> {
9048            let _ = self.started.send(());
9049            self.proceed.notified().await;
9050            anyhow::bail!("engine failed mid-turn")
9051        }
9052    }
9053
9054    #[tokio::test]
9055    async fn failed_turn_sweeps_pending_external_tool_calls() {
9056        let (started_tx, mut started_rx) = tokio::sync::mpsc::unbounded_channel();
9057        let proceed = Arc::new(tokio::sync::Notify::new());
9058        let mut builder = ExtensionRegistryBuilder::new();
9059        builder.inference_engine(Arc::new(SignalledFailureEngine {
9060            started: started_tx,
9061            proceed: proceed.clone(),
9062        }));
9063        let runtime =
9064            Arc::new(Runtime::new(builder.build().unwrap(), RuntimeConfig::default()).unwrap());
9065        let mut rx = runtime.subscribe_events();
9066        let turn_id = runtime
9067            .start_turn(StartTurnRequest {
9068                thread_id: "thread-sweep".to_string(),
9069                message: "go".to_string(),
9070                images: Vec::new(),
9071                provider_override: None,
9072                model_override: None,
9073                reasoning_override: None,
9074                workspace: test_workspace(),
9075                instructions: InstructionBundle {
9076                    system: None,
9077                    developer: None,
9078                    developer_context: None,
9079                },
9080                developer_context: None,
9081                task_ledger_required: false,
9082                service_tier_override: None,
9083            })
9084            .await
9085            .unwrap();
9086        tokio::time::timeout(std::time::Duration::from_secs(5), started_rx.recv())
9087            .await
9088            .unwrap()
9089            .unwrap();
9090
9091        let (tx, _pending_rx) = oneshot::channel();
9092        runtime.pending_external_tool_calls.lock().await.insert(
9093            "exttool-sweep-test".to_string(),
9094            PendingExternalToolCall {
9095                thread_id: "thread-sweep".to_string(),
9096                turn_id: turn_id.clone(),
9097                tool_id: "call-1".to_string(),
9098                tool_name: "acme_lookup".to_string(),
9099                tx,
9100            },
9101        );
9102        proceed.notify_one();
9103
9104        let outcome = tokio::time::timeout(std::time::Duration::from_secs(5), async {
9105            loop {
9106                let envelope = rx.recv().await.unwrap();
9107                if let RoderEvent::ExternalToolCallResolved(event) = envelope.event
9108                    && event.request_id == "exttool-sweep-test"
9109                {
9110                    break event.outcome;
9111                }
9112            }
9113        })
9114        .await
9115        .expect("turn failure must resolve pending external tool calls");
9116        assert_eq!(outcome, ExternalToolCallOutcome::Cancelled);
9117        assert!(runtime.pending_external_tool_calls.lock().await.is_empty());
9118    }
9119
9120    #[tokio::test]
9121    async fn thread_developer_instructions_layer_under_harness_prompt() {
9122        let requests = Arc::new(StdMutex::new(Vec::new()));
9123        let thread_root = std::env::temp_dir().join(format!(
9124            "roder-thread-instructions-{}",
9125            uuid::Uuid::new_v4()
9126        ));
9127        let mut builder = ExtensionRegistryBuilder::new();
9128        builder.inference_engine(Arc::new(SwitchCaptureEngine {
9129            requests: requests.clone(),
9130        }));
9131        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
9132            base_path: thread_root.clone(),
9133        }));
9134        builder.tool_contributor(Arc::new(ProfileToolContributor));
9135        let runtime =
9136            Arc::new(Runtime::new(builder.build().unwrap(), RuntimeConfig::default()).unwrap());
9137
9138        let request = captured_thread_override_request(
9139            &runtime,
9140            &requests,
9141            Vec::new(),
9142            Some("You are embedded in a host app.".to_string()),
9143            Vec::new(),
9144        )
9145        .await;
9146
9147        let system = request.instructions.system.expect("system instructions");
9148        assert!(system.starts_with("You are Roder"));
9149        let developer = request
9150            .instructions
9151            .developer
9152            .expect("developer instructions");
9153        assert!(developer.starts_with("You are embedded in a host app."));
9154
9155        let plain =
9156            captured_thread_override_request(&runtime, &requests, Vec::new(), None, Vec::new())
9157                .await;
9158        assert_eq!(plain.instructions.developer, None);
9159
9160        let _ = std::fs::remove_dir_all(thread_root);
9161    }
9162
9163    #[tokio::test]
9164    async fn thread_external_tools_are_advertised_and_shadow_builtins() {
9165        let requests = Arc::new(StdMutex::new(Vec::new()));
9166        let thread_root = std::env::temp_dir().join(format!(
9167            "roder-thread-external-tools-{}",
9168            uuid::Uuid::new_v4()
9169        ));
9170        let mut builder = ExtensionRegistryBuilder::new();
9171        builder.inference_engine(Arc::new(SwitchCaptureEngine {
9172            requests: requests.clone(),
9173        }));
9174        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
9175            base_path: thread_root.clone(),
9176        }));
9177        builder.tool_contributor(Arc::new(ProfileToolContributor));
9178        let runtime =
9179            Arc::new(Runtime::new(builder.build().unwrap(), RuntimeConfig::default()).unwrap());
9180
9181        let external_tools = vec![
9182            ToolSpec {
9183                name: "acme_lookup".to_string(),
9184                description: "Look up Acme workspace state.".to_string(),
9185                parameters: serde_json::json!({
9186                    "type": "object",
9187                    "properties": { "query": { "type": "string" } },
9188                    "required": ["query"]
9189                }),
9190            },
9191            ToolSpec {
9192                name: "edit".to_string(),
9193                description: "Host-managed edit.".to_string(),
9194                parameters: serde_json::json!({ "type": "object" }),
9195            },
9196        ];
9197        let request =
9198            captured_thread_override_request(&runtime, &requests, Vec::new(), None, external_tools)
9199                .await;
9200
9201        let acme = request
9202            .tools
9203            .iter()
9204            .find(|tool| tool.name == "acme_lookup")
9205            .expect("external tool advertised");
9206        assert_eq!(acme.description, "Look up Acme workspace state.");
9207        assert_eq!(acme.parameters["required"][0], "query");
9208        let edits = request
9209            .tools
9210            .iter()
9211            .filter(|tool| tool.name == "edit")
9212            .collect::<Vec<_>>();
9213        assert_eq!(edits.len(), 1, "external edit shadows the builtin");
9214        assert_eq!(edits[0].description, "Host-managed edit.");
9215
9216        let plain =
9217            captured_thread_override_request(&runtime, &requests, Vec::new(), None, Vec::new())
9218                .await;
9219        assert!(plain.tools.iter().all(|tool| tool.name != "acme_lookup"));
9220        let plain_edit = plain
9221            .tools
9222            .iter()
9223            .find(|tool| tool.name == "edit")
9224            .expect("builtin edit advertised on plain thread");
9225        assert_eq!(plain_edit.description, "edit test tool");
9226
9227        let _ = std::fs::remove_dir_all(thread_root);
9228    }
9229
9230    #[tokio::test]
9231    async fn model_switch_injects_summary_and_records_profile_segments() {
9232        let requests = Arc::new(StdMutex::new(Vec::new()));
9233        let thread_root = std::env::temp_dir().join(format!(
9234            "roder-model-switch-thread-{}",
9235            uuid::Uuid::new_v4()
9236        ));
9237        let mut builder = ExtensionRegistryBuilder::new();
9238        builder.inference_engine(Arc::new(SwitchCaptureEngine {
9239            requests: requests.clone(),
9240        }));
9241        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
9242            base_path: thread_root.clone(),
9243        }));
9244        builder.tool_contributor(Arc::new(ProfileToolContributor));
9245        let mut claude_profile = test_model_profile("claude-haiku-4-5-20251001");
9246        claude_profile.provider_family = ProviderFamily::Anthropic;
9247        claude_profile.edit_tool = Some(EDIT_TOOL_EDIT.to_string());
9248        let runtime = Arc::new(
9249            Runtime::new(
9250                builder.build().unwrap(),
9251                RuntimeConfig {
9252                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9253                    default_model: "gpt-5.5".to_string(),
9254                    model_profiles: std::collections::HashMap::from([
9255                        ("gpt-5.5".to_string(), test_model_profile("gpt-5.5")),
9256                        ("claude-haiku-4-5-20251001".to_string(), claude_profile),
9257                    ]),
9258                    ..RuntimeConfig::default()
9259                },
9260            )
9261            .unwrap(),
9262        );
9263        let thread_id = runtime
9264            .create_thread_with(CreateThreadRequest {
9265                title: Some("Model switch".to_string()),
9266                workspace: test_workspace(),
9267                workspace_id: None,
9268                root_id: None,
9269                provider: None,
9270                model: None,
9271                selection_mode: None,
9272                tool_allowlist: Vec::new(),
9273                developer_instructions: None,
9274                external_tools: Vec::new(),
9275                runner: None,
9276            })
9277            .await
9278            .unwrap()
9279            .thread_id;
9280        let mut rx = runtime.subscribe_events();
9281        for (message, model_override) in [
9282            ("first turn", None),
9283            ("second turn", Some("claude-haiku-4-5-20251001".to_string())),
9284        ] {
9285            let turn_id = runtime
9286                .start_turn(StartTurnRequest {
9287                    thread_id: thread_id.clone(),
9288                    message: message.to_string(),
9289                    images: Vec::new(),
9290                    provider_override: None,
9291                    model_override,
9292                    reasoning_override: None,
9293                    workspace: test_workspace(),
9294                    instructions: InstructionBundle::default(),
9295                    developer_context: None,
9296                    task_ledger_required: false,
9297                    service_tier_override: None,
9298                })
9299                .await
9300                .unwrap();
9301            tokio::time::timeout(std::time::Duration::from_secs(5), async {
9302                loop {
9303                    let envelope = rx.recv().await.unwrap();
9304                    if envelope.turn_id.as_deref() != Some(&turn_id) {
9305                        continue;
9306                    }
9307                    match envelope.event {
9308                        RoderEvent::TurnCompleted(_) => break,
9309                        RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9310                        _ => {}
9311                    }
9312                }
9313            })
9314            .await
9315            .unwrap();
9316        }
9317
9318        let captured = requests.lock().unwrap().clone();
9319        assert_eq!(captured.len(), 2);
9320        assert!(captured[1].transcript.iter().any(|item| {
9321            matches!(
9322                item,
9323                TranscriptItem::UserMessage(message)
9324                    if message.text.starts_with(MODEL_SWITCH_SUMMARY_PREFIX)
9325                        && message.text.contains("previous profile mock/gpt-5.5")
9326                        && message.text.contains("Current profile mock/claude-haiku-4-5-20251001")
9327                        && message.text.contains("Available tools now:")
9328            )
9329        }));
9330
9331        let snapshot = runtime
9332            .thread_store
9333            .as_ref()
9334            .unwrap()
9335            .load_thread(&thread_id)
9336            .await
9337            .unwrap()
9338            .unwrap();
9339        let trace_segments = snapshot
9340            .turns
9341            .iter()
9342            .flat_map(|turn| &turn.items)
9343            .filter(|item| {
9344                matches!(
9345                    item,
9346                    TranscriptItem::ProviderMetadata(value)
9347                        if value.get("kind").and_then(serde_json::Value::as_str)
9348                            == Some(MODEL_PROFILE_TRACE_KIND)
9349                            && value.get("segment").and_then(serde_json::Value::as_str)
9350                                == Some("assistant")
9351                )
9352            })
9353            .count();
9354        assert!(trace_segments >= 2);
9355        let _ = std::fs::remove_dir_all(thread_root);
9356    }
9357
9358    struct CountingTaskTool {
9359        calls: Arc<StdMutex<u32>>,
9360    }
9361
9362    #[async_trait::async_trait]
9363    impl ToolExecutor for CountingTaskTool {
9364        fn spec(&self) -> ToolSpec {
9365            ToolSpec {
9366                name: "task".to_string(),
9367                description: "Dispatch a test subagent.".to_string(),
9368                parameters: serde_json::json!({
9369                    "type": "object",
9370                    "properties": {
9371                        "description": { "type": "string" },
9372                        "prompt": { "type": "string" },
9373                        "parent_deadline_seconds": { "type": "integer" }
9374                    },
9375                    "required": ["description", "prompt"],
9376                    "additionalProperties": false
9377                }),
9378            }
9379        }
9380
9381        async fn execute(
9382            &self,
9383            _ctx: ToolExecutionContext,
9384            call: ToolCall,
9385        ) -> anyhow::Result<ToolResult> {
9386            *self.calls.lock().unwrap() += 1;
9387            Ok(ToolResult {
9388                id: call.id,
9389                name: call.name,
9390                text: "started child".to_string(),
9391                data: serde_json::json!({}),
9392                is_error: false,
9393            })
9394        }
9395    }
9396
9397    #[tokio::test]
9398    async fn runtime_profile_reaches_inference_request_and_turn_metadata() {
9399        let captured = Arc::new(StdMutex::new(None));
9400        let mut builder = ExtensionRegistryBuilder::new();
9401        builder.inference_engine(Arc::new(CapturingEngine {
9402            request: captured.clone(),
9403        }));
9404        let runtime = Arc::new(
9405            Runtime::new(
9406                builder.build().unwrap(),
9407                RuntimeConfig {
9408                    runtime_profile: RuntimeProfile::NonInteractive,
9409                    ..RuntimeConfig::default()
9410                },
9411            )
9412            .unwrap(),
9413        );
9414        let mut rx = runtime.subscribe_events();
9415        let turn_id = runtime
9416            .start_turn(StartTurnRequest {
9417                thread_id: "thread-profile".to_string(),
9418                message: "work unattended".to_string(),
9419                images: Vec::new(),
9420                provider_override: None,
9421                model_override: None,
9422                reasoning_override: None,
9423                workspace: test_workspace(),
9424                instructions: InstructionBundle {
9425                    system: None,
9426                    developer: Some("base developer".to_string()),
9427                    developer_context: None,
9428                },
9429                developer_context: None,
9430                task_ledger_required: false,
9431                service_tier_override: None,
9432            })
9433            .await
9434            .unwrap();
9435
9436        let mut observed_profile = None;
9437        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9438            loop {
9439                let envelope = rx.recv().await.unwrap();
9440                if envelope.turn_id.as_deref() != Some(&turn_id) {
9441                    continue;
9442                }
9443                match envelope.event {
9444                    RoderEvent::TurnStarted(event) => {
9445                        observed_profile = Some(event.runtime_profile);
9446                    }
9447                    RoderEvent::TurnCompleted(_) => break,
9448                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9449                    _ => {}
9450                }
9451            }
9452        })
9453        .await
9454        .unwrap();
9455
9456        assert_eq!(observed_profile, Some(RuntimeProfile::NonInteractive));
9457        let request = captured.lock().unwrap().clone().unwrap();
9458        assert_eq!(request.runtime.profile, RuntimeProfile::NonInteractive);
9459        let developer = request.instructions.developer.unwrap();
9460        assert!(developer.contains("base developer"));
9461        assert!(developer.contains("non-interactive profile"));
9462    }
9463
9464    #[tokio::test]
9465    async fn global_policy_mode_changes_do_not_create_runtime_thread_directory() {
9466        let workspace = runtime_test_workspace("global-policy-mode");
9467        let thread_root = workspace.join("threads");
9468        let mut builder = ExtensionRegistryBuilder::new();
9469        builder.inference_engine(Arc::new(FakeInferenceEngine));
9470        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
9471            base_path: thread_root.clone(),
9472        }));
9473        let runtime = Runtime::new(
9474            builder.build().unwrap(),
9475            RuntimeConfig {
9476                workspace: Some(workspace.display().to_string()),
9477                ..Default::default()
9478            },
9479        )
9480        .unwrap();
9481
9482        runtime
9483            .set_policy_mode(PolicyMode::AcceptAll, Some("test".to_string()))
9484            .await
9485            .unwrap();
9486
9487        assert!(!thread_root.join("runtime").exists());
9488        let _ = std::fs::remove_dir_all(workspace);
9489    }
9490
9491    #[tokio::test]
9492    async fn task_ledger_enforcement_injects_eval_reminder_before_work() {
9493        let captured = Arc::new(StdMutex::new(None));
9494        let mut builder = ExtensionRegistryBuilder::new();
9495        builder.inference_engine(Arc::new(CapturingEngine {
9496            request: captured.clone(),
9497        }));
9498        builder.tool_contributor(Arc::new(
9499            roder_ext_task_ledger::TaskLedgerToolContributor::default(),
9500        ));
9501        let runtime = Arc::new(
9502            Runtime::new(
9503                builder.build().unwrap(),
9504                RuntimeConfig {
9505                    runtime_profile: RuntimeProfile::Eval,
9506                    policy_mode: PolicyMode::Bypass,
9507                    agent_swarm_mode: false,
9508                    ultra_mode: false,
9509                    ..RuntimeConfig::default()
9510                },
9511            )
9512            .unwrap(),
9513        );
9514        let mut rx = runtime.subscribe_events();
9515        let turn_id = runtime
9516            .start_turn(StartTurnRequest {
9517                thread_id: "thread-ledger".to_string(),
9518                message: "decomposed work".to_string(),
9519                images: Vec::new(),
9520                provider_override: None,
9521                model_override: None,
9522                reasoning_override: None,
9523                workspace: test_workspace(),
9524                instructions: InstructionBundle::default(),
9525                developer_context: None,
9526                task_ledger_required: true,
9527                service_tier_override: None,
9528            })
9529            .await
9530            .unwrap();
9531
9532        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9533            loop {
9534                let envelope = rx.recv().await.unwrap();
9535                if envelope.turn_id.as_deref() == Some(&turn_id)
9536                    && matches!(envelope.event, RoderEvent::TurnCompleted(_))
9537                {
9538                    break;
9539                }
9540            }
9541        })
9542        .await
9543        .unwrap();
9544
9545        let request = captured.lock().unwrap().clone().unwrap();
9546        let developer = request.instructions.developer.unwrap();
9547        assert!(developer.contains("Task Ledger Required"));
9548        assert!(developer.contains("task_ledger.update"));
9549        let tool_names: Vec<_> = request
9550            .tools
9551            .iter()
9552            .map(|tool| tool.name.as_str())
9553            .collect();
9554        assert!(
9555            tool_names.contains(&TASK_LEDGER_TOOL_NAME),
9556            "tool names: {tool_names:?}"
9557        );
9558        assert_eq!(
9559            request.tool_choice,
9560            ToolChoice::Specific(TASK_LEDGER_TOOL_NAME.to_string())
9561        );
9562        assert_eq!(request.tools.len(), 1);
9563        assert_eq!(request.tools[0].name, TASK_LEDGER_TOOL_NAME);
9564    }
9565
9566    #[tokio::test]
9567    async fn eval_task_ledger_blocks_final_answer_until_open_items_are_completed() {
9568        let requests = Arc::new(StdMutex::new(Vec::new()));
9569        let mut builder = ExtensionRegistryBuilder::new();
9570        builder.inference_engine(Arc::new(TaskLedgerCompletionGateEngine {
9571            calls: StdMutex::new(0),
9572            requests: requests.clone(),
9573        }));
9574        builder.tool_contributor(Arc::new(
9575            roder_ext_task_ledger::TaskLedgerToolContributor::default(),
9576        ));
9577        let runtime = Arc::new(
9578            Runtime::new(
9579                builder.build().unwrap(),
9580                RuntimeConfig {
9581                    runtime_profile: RuntimeProfile::Eval,
9582                    policy_mode: PolicyMode::Bypass,
9583                    agent_swarm_mode: false,
9584                    ultra_mode: false,
9585                    ..RuntimeConfig::default()
9586                },
9587            )
9588            .unwrap(),
9589        );
9590        let mut rx = runtime.subscribe_events();
9591        let turn_id = runtime
9592            .start_turn(StartTurnRequest {
9593                thread_id: "thread-ledger-completion".to_string(),
9594                message: "write the answer file".to_string(),
9595                images: Vec::new(),
9596                provider_override: None,
9597                model_override: None,
9598                reasoning_override: None,
9599                workspace: test_workspace(),
9600                instructions: InstructionBundle::default(),
9601                developer_context: None,
9602                task_ledger_required: true,
9603                service_tier_override: None,
9604            })
9605            .await
9606            .unwrap();
9607
9608        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9609            loop {
9610                let envelope = rx.recv().await.unwrap();
9611                if envelope.turn_id.as_deref() != Some(&turn_id) {
9612                    continue;
9613                }
9614                match envelope.event {
9615                    RoderEvent::TurnCompleted(_) => break,
9616                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9617                    _ => {}
9618                }
9619            }
9620        })
9621        .await
9622        .unwrap();
9623
9624        let requests = requests.lock().unwrap().clone();
9625        assert_eq!(requests.len(), 4);
9626        assert!(requests[2].transcript.iter().any(|item| {
9627            matches!(
9628                item,
9629                TranscriptItem::UserMessage(message)
9630                    if message.text.contains("Task Ledger Completion Required")
9631                        && message.text.contains("Write /app/result.txt")
9632            )
9633        }));
9634        assert!(requests[3].transcript.iter().any(|item| {
9635            matches!(
9636                item,
9637                TranscriptItem::ToolResult(result)
9638                    if result.name.as_deref() == Some(TASK_LEDGER_TOOL_NAME)
9639                        && result.result.contains("Task ledger: 2/2 completed")
9640            )
9641        }));
9642    }
9643
9644    #[tokio::test]
9645    async fn eval_task_ledger_checkpoint_requests_scoreable_file_before_final_reserve() {
9646        let requests = Arc::new(StdMutex::new(Vec::new()));
9647        let mut builder = ExtensionRegistryBuilder::new();
9648        builder.inference_engine(Arc::new(TaskLedgerCompletionGateEngine {
9649            calls: StdMutex::new(0),
9650            requests: requests.clone(),
9651        }));
9652        builder.tool_contributor(Arc::new(
9653            roder_ext_task_ledger::TaskLedgerToolContributor::default(),
9654        ));
9655        let runtime = Arc::new(
9656            Runtime::new(
9657                builder.build().unwrap(),
9658                RuntimeConfig {
9659                    runtime_profile: RuntimeProfile::Eval,
9660                    policy_mode: PolicyMode::Bypass,
9661                    agent_swarm_mode: false,
9662                    ultra_mode: false,
9663                    turn_deadline_seconds: Some(120),
9664                    ..RuntimeConfig::default()
9665                },
9666            )
9667            .unwrap(),
9668        );
9669        let mut rx = runtime.subscribe_events();
9670        let turn_id = runtime
9671            .start_turn(StartTurnRequest {
9672                thread_id: "thread-ledger-checkpoint".to_string(),
9673                message: "write the answer file".to_string(),
9674                images: Vec::new(),
9675                provider_override: None,
9676                model_override: None,
9677                reasoning_override: None,
9678                workspace: test_workspace(),
9679                instructions: InstructionBundle::default(),
9680                developer_context: None,
9681                task_ledger_required: true,
9682                service_tier_override: None,
9683            })
9684            .await
9685            .unwrap();
9686
9687        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9688            loop {
9689                let envelope = rx.recv().await.unwrap();
9690                if envelope.turn_id.as_deref() != Some(&turn_id) {
9691                    continue;
9692                }
9693                match envelope.event {
9694                    RoderEvent::TurnCompleted(_) => break,
9695                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9696                    _ => {}
9697                }
9698            }
9699        })
9700        .await
9701        .unwrap();
9702
9703        let requests = requests.lock().unwrap().clone();
9704        assert!(requests.len() >= 2);
9705        assert!(requests[1].transcript.iter().any(|item| {
9706            matches!(
9707                item,
9708                TranscriptItem::UserMessage(message)
9709                    if message.text.contains("Scoreable Output Checkpoint")
9710                        && message.text.contains("ensure the required output file(s) exist")
9711                        && message.text.contains("Write /app/result.txt")
9712            )
9713        }));
9714    }
9715
9716    #[test]
9717    fn deadline_task_ledger_prompt_preserves_scoreable_work_instruction() {
9718        let prompt = task_ledger_deadline_completion_prompt(
9719            12,
9720            30,
9721            "Task Ledger Completion Required: write /app/result.txt, then call task_ledger.update",
9722        );
9723
9724        assert!(prompt.contains("12 seconds remain"));
9725        assert!(prompt.contains("create or update the required scoreable output files"));
9726        assert!(prompt.contains("write /app/result.txt"));
9727        assert!(prompt.contains(TASK_LEDGER_TOOL_NAME));
9728    }
9729
9730    #[test]
9731    fn scoreable_checkpoint_prompt_preserves_provisional_file_instruction() {
9732        let prompt = task_ledger_scoreable_checkpoint_prompt(
9733            120,
9734            "Task Ledger Completion Required: write /app/result.txt, then call task_ledger.update",
9735        );
9736
9737        assert!(prompt.contains("120 seconds remain"));
9738        assert!(prompt.contains("best evidence-backed answer"));
9739        assert!(prompt.contains("even if provisional"));
9740        assert!(prompt.contains("preserve that candidate"));
9741        assert!(prompt.contains("partial-coverage"));
9742        assert!(prompt.contains("write /app/result.txt"));
9743        assert!(prompt.contains(TASK_LEDGER_TOOL_NAME));
9744    }
9745
9746    #[test]
9747    fn open_task_ledger_moves_inference_timeout_to_scoreable_checkpoint() {
9748        let deadline = Some(OffsetDateTime::now_utc() + Duration::seconds(870));
9749        let transcript = vec![TranscriptItem::ToolResult(ToolResultRecord {
9750            id: "ledger-open".to_string(),
9751            name: Some(TASK_LEDGER_TOOL_NAME.to_string()),
9752            result: "Task ledger: 0/1 completed\n- pending: Write /app/result.txt [write]"
9753                .to_string(),
9754            display_payload: None,
9755            is_error: false,
9756        })];
9757
9758        let (_, action) = inference_timeout_deadline(
9759            deadline,
9760            RuntimeProfile::Eval,
9761            true,
9762            30,
9763            false,
9764            0,
9765            &transcript,
9766        )
9767        .unwrap();
9768
9769        assert_eq!(action, InferenceTimeoutAction::ScoreableCheckpoint);
9770    }
9771
9772    #[tokio::test]
9773    async fn verification_gate_forces_eval_code_changes_through_review() {
9774        let mut builder = ExtensionRegistryBuilder::new();
9775        builder.inference_engine(Arc::new(VerificationGateEngine {
9776            calls: StdMutex::new(0),
9777        }));
9778        builder.tool_contributor(Arc::new(WriteFileContributor));
9779        builder.tool_contributor(Arc::new(
9780            roder_ext_verification::VerificationToolContributor,
9781        ));
9782        let runtime = Arc::new(
9783            Runtime::new(
9784                builder.build().unwrap(),
9785                RuntimeConfig {
9786                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9787                    default_model: "mock".to_string(),
9788                    runtime_profile: RuntimeProfile::Eval,
9789                    policy_mode: PolicyMode::Bypass,
9790                    agent_swarm_mode: false,
9791                    ultra_mode: false,
9792                    ..RuntimeConfig::default()
9793                },
9794            )
9795            .unwrap(),
9796        );
9797        let mut rx = runtime.subscribe_events();
9798        let turn_id = runtime
9799            .start_turn(StartTurnRequest {
9800                thread_id: "thread-verification".to_string(),
9801                message: "write code".to_string(),
9802                images: Vec::new(),
9803                provider_override: None,
9804                model_override: None,
9805                reasoning_override: None,
9806                workspace: test_workspace(),
9807                instructions: InstructionBundle::default(),
9808                developer_context: None,
9809                task_ledger_required: false,
9810                service_tier_override: None,
9811            })
9812            .await
9813            .unwrap();
9814
9815        let mut saw_required = false;
9816        let mut saw_completed = false;
9817        let mut final_text = String::new();
9818        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9819            loop {
9820                let envelope = rx.recv().await.unwrap();
9821                if envelope.turn_id.as_deref() != Some(&turn_id) {
9822                    continue;
9823                }
9824                match envelope.event {
9825                    RoderEvent::VerificationRequired(event) => {
9826                        saw_required = true;
9827                        assert_eq!(event.changed_files, vec!["src/lib.rs"]);
9828                    }
9829                    RoderEvent::VerificationCompleted(event) => {
9830                        saw_completed = true;
9831                        assert!(event.passed);
9832                    }
9833                    RoderEvent::InferenceEventReceived(event) => {
9834                        if let InferenceEvent::MessageDelta(delta) = event.event {
9835                            final_text.push_str(&delta.text);
9836                        }
9837                    }
9838                    RoderEvent::TurnCompleted(_) => break,
9839                    _ => {}
9840                }
9841            }
9842        })
9843        .await
9844        .unwrap();
9845
9846        assert!(saw_required);
9847        assert!(saw_completed);
9848        assert!(final_text.contains("verified final"));
9849    }
9850
9851    #[tokio::test]
9852    async fn speed_policy_changes_reasoning_across_eval_model_calls_without_model_switch() {
9853        let requests = Arc::new(StdMutex::new(Vec::new()));
9854        let mut builder = ExtensionRegistryBuilder::new();
9855        builder.inference_engine(Arc::new(SpeedPolicyEngine {
9856            calls: StdMutex::new(0),
9857            requests: requests.clone(),
9858        }));
9859        builder.tool_contributor(Arc::new(WriteFileContributor));
9860        builder.tool_contributor(Arc::new(
9861            roder_ext_verification::VerificationToolContributor,
9862        ));
9863        let runtime = Arc::new(
9864            Runtime::new(
9865                builder.build().unwrap(),
9866                RuntimeConfig {
9867                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9868                    default_model: "gpt-5.5".to_string(),
9869                    runtime_profile: RuntimeProfile::Eval,
9870                    policy_mode: PolicyMode::Bypass,
9871                    agent_swarm_mode: false,
9872                    ultra_mode: false,
9873                    ..RuntimeConfig::default()
9874                },
9875            )
9876            .unwrap(),
9877        );
9878        let mut rx = runtime.subscribe_events();
9879        let turn_id = runtime
9880            .start_turn(StartTurnRequest {
9881                thread_id: "thread-speed-policy".to_string(),
9882                message: "write code".to_string(),
9883                images: Vec::new(),
9884                provider_override: None,
9885                model_override: None,
9886                reasoning_override: None,
9887                workspace: test_workspace(),
9888                instructions: InstructionBundle::default(),
9889                developer_context: None,
9890                task_ledger_required: false,
9891                service_tier_override: None,
9892            })
9893            .await
9894            .unwrap();
9895
9896        let mut saw_speed_policy_event = false;
9897        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9898            loop {
9899                let envelope = rx.recv().await.unwrap();
9900                if envelope.turn_id.as_deref() != Some(&turn_id) {
9901                    continue;
9902                }
9903                match envelope.event {
9904                    RoderEvent::InferenceStarted(event) => {
9905                        if event.speed_policy.is_some() {
9906                            saw_speed_policy_event = true;
9907                        }
9908                    }
9909                    RoderEvent::TurnCompleted(_) => break,
9910                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9911                    _ => {}
9912                }
9913            }
9914        })
9915        .await
9916        .unwrap();
9917
9918        let requests = requests.lock().unwrap().clone();
9919        assert!(saw_speed_policy_event);
9920        assert!(requests.len() >= 4);
9921        assert!(requests.iter().all(|request| {
9922            request.model.provider == roder_api::catalog::PROVIDER_MOCK
9923                && request.model.model == "gpt-5.5"
9924        }));
9925        assert_eq!(
9926            requests[0].runtime.speed_policy.as_ref().map(|d| d.phase),
9927            Some(roder_api::inference::SpeedPolicyPhase::Orientation)
9928        );
9929        assert_eq!(requests[0].reasoning.level.as_deref(), Some(REASONING_HIGH));
9930        assert_eq!(
9931            requests[1].runtime.speed_policy.as_ref().map(|d| d.phase),
9932            Some(roder_api::inference::SpeedPolicyPhase::Execution)
9933        );
9934        assert_eq!(requests[1].reasoning.level.as_deref(), Some(REASONING_LOW));
9935        assert_eq!(
9936            requests[2].runtime.speed_policy.as_ref().map(|d| d.phase),
9937            Some(roder_api::inference::SpeedPolicyPhase::Verification)
9938        );
9939        assert_eq!(requests[2].reasoning.level.as_deref(), Some(REASONING_HIGH));
9940        assert_eq!(
9941            requests[2]
9942                .metadata
9943                .pointer("/speedPolicy/phase")
9944                .and_then(serde_json::Value::as_str),
9945            Some("verification")
9946        );
9947    }
9948
9949    #[tokio::test]
9950    async fn deadline_turn_timeout_emits_partial_result_and_clears_active_turn() {
9951        let mut builder = ExtensionRegistryBuilder::new();
9952        builder.inference_engine(Arc::new(DeadlineEngine));
9953        let runtime = Arc::new(
9954            Runtime::new(
9955                builder.build().unwrap(),
9956                RuntimeConfig {
9957                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9958                    default_model: "mock".to_string(),
9959                    runtime_profile: RuntimeProfile::Eval,
9960                    turn_deadline_seconds: Some(1),
9961                    ..RuntimeConfig::default()
9962                },
9963            )
9964            .unwrap(),
9965        );
9966        let mut rx = runtime.subscribe_events();
9967        let turn_id = runtime
9968            .start_turn(StartTurnRequest {
9969                thread_id: "thread-deadline".to_string(),
9970                message: "slow work".to_string(),
9971                images: Vec::new(),
9972                provider_override: None,
9973                model_override: None,
9974                reasoning_override: None,
9975                workspace: test_workspace(),
9976                instructions: InstructionBundle::default(),
9977                developer_context: None,
9978                task_ledger_required: false,
9979                service_tier_override: None,
9980            })
9981            .await
9982            .unwrap();
9983
9984        let mut saw_partial = false;
9985        let mut saw_deadline = false;
9986        let mut failed_kind = None;
9987        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9988            loop {
9989                let envelope = rx.recv().await.unwrap();
9990                if envelope.turn_id.as_deref() != Some(&turn_id) {
9991                    continue;
9992                }
9993                match envelope.event {
9994                    RoderEvent::TurnPartialResult(event) => {
9995                        saw_partial = event.summary.contains("partial turn state");
9996                    }
9997                    RoderEvent::TurnDeadlineExceeded(event) => {
9998                        saw_deadline = event.partial_result.contains("transcript items");
9999                    }
10000                    RoderEvent::TurnFailed(event) => {
10001                        failed_kind = event.error_kind;
10002                        break;
10003                    }
10004                    _ => {}
10005                }
10006            }
10007        })
10008        .await
10009        .unwrap();
10010
10011        for _ in 0..20 {
10012            if !runtime.active_turns.read().await.contains_key(&turn_id) {
10013                break;
10014            }
10015            tokio::time::sleep(std::time::Duration::from_millis(10)).await;
10016        }
10017        assert!(saw_partial);
10018        assert!(saw_deadline);
10019        assert_eq!(failed_kind.as_deref(), Some("deadline_timeout"));
10020        assert!(!runtime.active_turns.read().await.contains_key(&turn_id));
10021    }
10022
10023    #[tokio::test]
10024    async fn deadline_skips_subagent_task_when_remaining_budget_is_too_low() {
10025        let calls = Arc::new(StdMutex::new(0));
10026        let mut builder = ExtensionRegistryBuilder::new();
10027        builder.inference_engine(Arc::new(CapturingEngine {
10028            request: Arc::new(StdMutex::new(None)),
10029        }));
10030        let task_tool = Arc::new(CountingTaskTool {
10031            calls: calls.clone(),
10032        });
10033        builder.tool_contributor(Arc::new(TestToolContributor { tool: task_tool }));
10034        let runtime = Arc::new(
10035            Runtime::new(
10036                builder.build().unwrap(),
10037                RuntimeConfig {
10038                    policy_mode: PolicyMode::Bypass,
10039                    agent_swarm_mode: false,
10040                    ultra_mode: false,
10041                    ..RuntimeConfig::default()
10042                },
10043            )
10044            .unwrap(),
10045        );
10046
10047        let result = runtime
10048            .route_tool_call(
10049                &"thread-deadline-task".to_string(),
10050                &"turn-deadline-task".to_string(),
10051                ToolCallCompleted {
10052                    id: "task-1".to_string(),
10053                    name: "task".to_string(),
10054                    arguments: serde_json::json!({
10055                        "description": "inspect",
10056                        "prompt": "read"
10057                    })
10058                    .to_string(),
10059                },
10060                None,
10061                Some(OffsetDateTime::now_utc() + Duration::seconds(1)),
10062            )
10063            .await
10064            .unwrap();
10065
10066        assert!(result.is_error);
10067        assert!(result.result.contains("deadline policy skipped"));
10068        assert_eq!(*calls.lock().unwrap(), 0);
10069    }
10070
10071    struct TestToolContributor {
10072        tool: Arc<dyn ToolExecutor>,
10073    }
10074
10075    impl ToolContributor for TestToolContributor {
10076        fn id(&self) -> String {
10077            "test-tool".to_string()
10078        }
10079
10080        fn contribute(&self, registry: &mut ToolRegistry) -> anyhow::Result<()> {
10081            registry.register(self.tool.clone())
10082        }
10083    }
10084
10085    #[tokio::test]
10086    async fn agent_swarm_mode_override_is_per_thread() {
10087        let runtime = Runtime::fake().unwrap();
10088        let trigger = roder_api::subagents::AgentSwarmModeTrigger::Manual;
10089
10090        // Off everywhere by default.
10091        assert!(
10092            !runtime
10093                .effective_agent_swarm_mode_for_thread("thread-a")
10094                .await
10095        );
10096        assert!(
10097            !runtime
10098                .effective_agent_swarm_mode_for_thread("thread-b")
10099                .await
10100        );
10101
10102        // Enabling on thread-a does not leak into thread-b.
10103        assert!(
10104            runtime
10105                .set_agent_swarm_mode_for_thread("thread-a", true, trigger)
10106                .await
10107        );
10108        assert!(
10109            runtime
10110                .effective_agent_swarm_mode_for_thread("thread-a")
10111                .await
10112        );
10113        assert!(
10114            !runtime
10115                .effective_agent_swarm_mode_for_thread("thread-b")
10116                .await
10117        );
10118
10119        // A per-thread `off` override wins over a runtime-global `on` default.
10120        runtime.set_agent_swarm_mode(true, trigger).await.unwrap();
10121        runtime
10122            .set_agent_swarm_mode_for_thread("thread-b", false, trigger)
10123            .await;
10124        assert!(
10125            !runtime
10126                .effective_agent_swarm_mode_for_thread("thread-b")
10127                .await,
10128            "explicit per-thread off overrides the global on default"
10129        );
10130        // A thread with no override still follows the global default.
10131        assert!(
10132            runtime
10133                .effective_agent_swarm_mode_for_thread("thread-c")
10134                .await,
10135            "threads without an override follow the runtime-global default"
10136        );
10137    }
10138
10139    #[tokio::test]
10140    async fn set_agent_swarm_mode_for_thread_emits_event_with_real_thread_id() {
10141        let runtime = Runtime::fake().unwrap();
10142        let mut events = runtime.subscribe_events();
10143        runtime
10144            .set_agent_swarm_mode_for_thread(
10145                "thread-xyz",
10146                true,
10147                roder_api::subagents::AgentSwarmModeTrigger::Task,
10148            )
10149            .await;
10150        let mut saw = false;
10151        for _ in 0..8 {
10152            let envelope = tokio::time::timeout(std::time::Duration::from_secs(2), events.recv())
10153                .await
10154                .unwrap()
10155                .unwrap();
10156            if let RoderEvent::AgentSwarmModeChanged(event) = envelope.event {
10157                assert_eq!(event.thread_id, "thread-xyz");
10158                assert!(event.enabled);
10159                assert_eq!(
10160                    event.trigger,
10161                    roder_api::subagents::AgentSwarmModeTrigger::Task
10162                );
10163                saw = true;
10164                break;
10165            }
10166        }
10167        assert!(
10168            saw,
10169            "expected an AgentSwarmModeChanged event for the thread"
10170        );
10171    }
10172
10173    #[tokio::test]
10174    async fn ultra_mode_override_is_per_thread() {
10175        let runtime = Runtime::fake().unwrap();
10176        let trigger = roder_api::subagents::UltraModeTrigger::Manual;
10177
10178        assert!(!runtime.effective_ultra_mode_for_thread("thread-a").await);
10179        assert!(!runtime.effective_ultra_mode_for_thread("thread-b").await);
10180
10181        assert!(
10182            runtime
10183                .set_ultra_mode_for_thread("thread-a", true, trigger)
10184                .await
10185        );
10186        assert!(runtime.effective_ultra_mode_for_thread("thread-a").await);
10187        assert!(!runtime.effective_ultra_mode_for_thread("thread-b").await);
10188
10189        runtime.set_ultra_mode(true, trigger).await.unwrap();
10190        runtime
10191            .set_ultra_mode_for_thread("thread-b", false, trigger)
10192            .await;
10193        assert!(
10194            !runtime.effective_ultra_mode_for_thread("thread-b").await,
10195            "explicit per-thread off overrides the global on default"
10196        );
10197        assert!(
10198            runtime.effective_ultra_mode_for_thread("thread-c").await,
10199            "threads without an override follow the runtime-global default"
10200        );
10201    }
10202
10203    #[tokio::test]
10204    async fn set_ultra_mode_for_thread_emits_event_with_real_thread_id() {
10205        let runtime = Runtime::fake().unwrap();
10206        let mut events = runtime.subscribe_events();
10207        runtime
10208            .set_ultra_mode_for_thread(
10209                "thread-ultra",
10210                true,
10211                roder_api::subagents::UltraModeTrigger::Task,
10212            )
10213            .await;
10214        let mut saw = false;
10215        for _ in 0..8 {
10216            let envelope = tokio::time::timeout(std::time::Duration::from_secs(2), events.recv())
10217                .await
10218                .unwrap()
10219                .unwrap();
10220            if let RoderEvent::UltraModeChanged(event) = envelope.event {
10221                assert_eq!(event.thread_id, "thread-ultra");
10222                assert!(event.enabled);
10223                assert_eq!(event.trigger, roder_api::subagents::UltraModeTrigger::Task);
10224                saw = true;
10225                break;
10226            }
10227        }
10228        assert!(saw, "expected an UltraModeChanged event for the thread");
10229    }
10230}