Skip to main content

roder_core/
runtime.rs

1#[path = "runtime/sampling.rs"]
2pub(crate) mod sampling;
3#[path = "tool_advertisement.rs"]
4mod tool_advertisement;
5use sampling::{
6    SamplingPreempted, SamplingRetry, completed_call_replayed, preemptible, wait_for_steer,
7};
8#[path = "runtime/sampling_output.rs"]
9mod sampling_output;
10use sampling_output::{OutputContext, SamplingOutput};
11mod compaction_template;
12#[path = "runtime/eager_tools.rs"]
13mod eager_tools;
14mod execution_receipts;
15mod owner_handoff;
16mod service_tier;
17mod thread_admission;
18mod thread_policy;
19mod thread_unload;
20#[path = "runtime/tool_approvals.rs"]
21mod tool_approvals;
22#[path = "runtime/turn_start.rs"]
23mod turn_start;
24#[path = "runtime/usage_accounting.rs"]
25mod usage_accounting;
26use eager_tools::EagerTools;
27#[path = "runtime/patch_progress.rs"]
28mod patch_progress;
29use patch_progress::PatchProgressTracker;
30
31use std::collections::{HashMap, HashSet};
32use std::path::PathBuf;
33use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
34use std::sync::{Arc, Weak};
35
36use anyhow::Context;
37use futures::StreamExt;
38use futures::future::{AbortHandle, Abortable, BoxFuture, try_join_all};
39use roder_api::catalog::{
40    EDIT_TOOL_EDIT, EDIT_TOOL_PATCH, PROVIDER_GEMINI, REASONING_NONE, REASONING_ULTRA,
41    built_in_model_profile, built_in_model_profile_for_provider, lookup_model,
42    model_supports_reasoning_effort,
43};
44use roder_api::context::PolicyGate;
45use roder_api::events::*;
46use roder_api::extension::ExtensionRegistry;
47use roder_api::inference::{
48    AgentInferenceRequest, HostedWebSearchConfig, HostedWebSearchMode, InferenceEngine,
49    InferenceEvent, InferenceTurnContext, InstructionBundle, ModelHarnessProfile,
50    ModelSchemaPolicy, ModelSelection, OutputConfig, ProviderTurnCleanup, ReasoningConfig,
51    RuntimeHints, RuntimeProfile, TokenUsage, ToolCallCompleted, ToolSearchConfig,
52    ToolSearchConfigOverlay, finish_reason_from_stop_reason,
53};
54use roder_api::inference_routing::{InferenceRoutingOutcome, ModelSelectionMode};
55use roder_api::lifecycle::{
56    LifecycleMetricsSnapshot, TurnCleanupOwnership, TurnCleanupState, TurnLifecycleReason,
57    TurnLifecycleRecord, TurnLifecycleSnapshot, TurnLifecycleState, turn_lifecycle_snapshot,
58};
59use roder_api::policy_mode::{PolicyDecision, PolicyMode};
60use roder_api::reliability::{
61    ReliabilityContext, ReliabilityDetails, ReliabilityErrorClass, ReliabilityLimitDecision,
62    ReliabilityLimitRecorded, ReliabilityRequestPolicy, ReliabilityRetryDecision,
63    ReliabilityRetryRecorded, provider_retry_delay_ms,
64};
65use roder_api::remote_runner::{
66    RemoteRunnerProvider, RemoteRunnerSession, RemoteWorkspace, RunnerDestination,
67    RunnerSessionState, ThreadRunnerBinding,
68};
69use roder_api::subagents::SubagentDefinition;
70use roder_api::teams::{
71    TeamId, TeamMailboxMessage, TeamMailboxMessageKind, TeamMemberDescriptor, TeamMemberRole,
72    TeamMemberStatus,
73};
74use roder_api::thread::{
75    ThreadItemEvent, ThreadItemEventKind, ThreadMetadata, ThreadSnapshot, ThreadStore,
76    ThreadUsageMetadata, is_synthetic_event_thread_id, validate_thread_workspace,
77};
78use roder_api::tools::{ToolCall, ToolChoice, ToolExecutionContext, ToolRegistry, ToolResult};
79use roder_api::transcript::{
80    AssistantMessage, ErrorRecord, InputImage, ReasoningSummary, ToolCallRecord, ToolResultRecord,
81    TranscriptItem, UserMessage,
82};
83use roder_sandbox::ScopedFilesystem;
84use roder_sandbox::process::LocalProcessRunner;
85use roder_skills::{SkillRegistry, SkillRegistryOptions};
86use time::{Duration, OffsetDateTime};
87use tokio::sync::{Mutex, Notify, RwLock, oneshot};
88
89mod codex_v2;
90
91use crate::artifacts::{
92    ContextArtifactStore as FilesystemContextArtifactStore, default_context_artifact_dir,
93};
94use crate::browser_routing::apply_browser_routing;
95use crate::bus::EventBus;
96use crate::dynamic_workflows::{
97    DynamicWorkflowEffortProfile, RuntimeDynamicWorkflowConfig, WorkflowTriggerDecision,
98    classify_workflow_trigger, ultracode_reasoning_level_for_model,
99};
100use crate::fake_provider::FakeInferenceEngine;
101use crate::goals::RuntimeGoalController;
102use crate::inference_routing::{
103    InferenceRoutingRequest, RuntimeInferenceRouterConfig, collect_inference_routing_candidates,
104    route_inference_selection, transcript_failure_count_since,
105};
106use crate::instructions::{
107    apply_agent_swarm_mode, apply_codex_multi_agent_mode, apply_model_instruction_overlay,
108    apply_parallel_web_tools, apply_plan_mode, apply_runtime_profile, apply_task_ledger_required,
109    apply_thread_developer_instructions, apply_turn_developer_context,
110};
111use crate::policy_gate::DefaultPolicyGate;
112use crate::reliability::{
113    ReliabilityLimitHit, RuntimeReliabilityConfig, TurnReliabilityState,
114    provider_stream_retry_cause,
115};
116pub use crate::speed_policy::RuntimeSpeedPolicyConfig;
117use crate::speed_policy::{SpeedPolicyState, reasoning_from_decision};
118use crate::subagent_traces::{RuntimeAgentSwarmProgressSink, RuntimeSubagentTraceSink};
119use crate::teams::{TeamManager, TeamMemberStartRequest, TeamStartRequest, TeamState};
120use crate::thread_item_cache::{ThreadItemCache, ThreadItemCacheEntry};
121use crate::tool_result_images::ToolImages;
122use crate::verification_gate::VerificationGateState;
123
124const MAX_TOOL_ROUNDS_PER_TURN: usize = 1024;
125/// Injected when `continue_on_failure_limit` turns a consecutive-tool-failure
126/// limit into a continuation, or as the empty-tool-call persistence nudge, so
127/// the model keeps working instead of ending the turn early.
128const RELIABILITY_CONTINUATION_PROMPT: &str = "Verify the task is fully complete. If any part remains unfinished or unverified, keep working using the available tools — try an alternative approach if one is failing. Only stop once the solution is complete and verified; if it is already complete, restate your final answer.";
129/// Capacity of the runtime event broadcast ring buffer. The TUI drains this on
130/// its render loop, which during an active turn only wakes at the ~6 FPS status
131/// animation cadence (backend events do not wake the input poll), so up to
132/// ~166ms of events buffer between drains. A reasoning-high provider that runs
133/// its whole tool loop inside one turn (e.g. claude-code) streams a firehose of
134/// `InferenceEventReceived` deltas plus per-step tool/thinking events; the old
135/// 1024-slot buffer overflowed in those windows and silently dropped tool and
136/// thinking rows from the live view. Sized for generous headroom across bursts
137/// and brief render stalls.
138const EVENT_BUS_CAPACITY: usize = 16_384;
139
140/// Turn id reported to session-scoped local hooks. `SessionStart` and
141/// `SessionEnd` bracket the session rather than any one turn, but the hook
142/// payload carries a turn id, so they share this synthetic value.
143const SESSION_HOOK_TURN_ID: &str = "session";
144pub(crate) const FINAL_ANSWER_PHASE: &str = "final_answer";
145pub(crate) const TASK_LEDGER_TOOL_NAME: &str = "task_ledger.update";
146const TASK_LEDGER_COMPLETION_REMINDER_LIMIT: u8 = 2;
147const TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS: u64 = 180;
148const TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT: u8 = 1;
149pub(crate) const MIN_CHILD_DEADLINE_SECONDS: u64 = 2;
150const MODEL_PROFILE_TRACE_KIND: &str = "model_profile_segment";
151const MODEL_SWITCH_SUMMARY_PREFIX: &str = "Model switch summary:";
152const PROVIDER_CLEANUP_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5);
153
154#[derive(Clone, Copy, Debug, Eq, PartialEq)]
155enum InferenceTimeoutAction {
156    ScoreableCheckpoint,
157    Finalization,
158}
159
160#[derive(Debug, Clone)]
161pub struct RuntimeConfig {
162    pub default_provider: String,
163    pub default_model: String,
164    pub reasoning: Option<String>,
165    pub auto_compact_token_limit: Option<u32>,
166    pub file_backed_dynamic_context: bool,
167    pub hosted_web_search: HostedWebSearchConfig,
168    pub tool_search: ToolSearchConfig,
169    pub provider_tool_search: HashMap<String, ToolSearchConfigOverlay>,
170    pub model_tool_search: HashMap<String, ToolSearchConfigOverlay>,
171    pub model_edit_tools: HashMap<String, String>,
172    pub model_parallel_tool_calls: HashMap<String, bool>,
173    pub model_profiles: HashMap<String, ModelHarnessProfile>,
174    pub tool_allowlist: Vec<String>,
175    /// Seconds a host-executed external tool call may stay unresolved before it fails with a timeout error.
176    pub external_tool_timeout_seconds: u64,
177    pub command_shell: String,
178    pub workspace: Option<String>,
179    pub policy_mode: PolicyMode,
180    /// Whether agent-swarm mode is active (roadmap 104). When on, the runtime
181    /// injects the swarm reminder into each turn's developer instructions so any
182    /// client benefits, not just the TUI.
183    pub agent_swarm_mode: bool,
184    /// Whether ultra mode is active. When on, the runtime injects proactive
185    /// multi-agent delegation policy for any model (not only Codex Sol/Terra
186    /// Ultra effort). Selecting Ultra reasoning on Sol/Terra still enables
187    /// proactive multi-agent for that model without requiring this flag.
188    pub ultra_mode: bool,
189    pub runtime_profile: RuntimeProfile,
190    pub inference_router: RuntimeInferenceRouterConfig,
191    pub speed_policy: RuntimeSpeedPolicyConfig,
192    pub dynamic_workflows: RuntimeDynamicWorkflowConfig,
193    pub reliability: RuntimeReliabilityConfig,
194    /// Positive wall-clock limit for a turn in any runtime profile. `None` or zero is unbounded.
195    pub turn_deadline_seconds: Option<u64>,
196    pub remote_runner_destination: Option<RunnerDestination>,
197    pub team_data_dir: Option<PathBuf>,
198    pub roadmap_data_dir: Option<PathBuf>,
199    pub media_generation: crate::media_generation::RuntimeMediaGenerationConfig,
200    /// `[review]` settings: the review sub-turn and its publishers.
201    pub review: crate::review::RuntimeReviewConfig,
202}
203
204impl Default for RuntimeConfig {
205    fn default() -> Self {
206        Self {
207            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
208            default_model: "mock".to_string(),
209            reasoning: None,
210            auto_compact_token_limit: None,
211            file_backed_dynamic_context: true,
212            hosted_web_search: HostedWebSearchConfig::cached(),
213            tool_search: ToolSearchConfig::default(),
214            provider_tool_search: HashMap::new(),
215            model_tool_search: HashMap::new(),
216            model_edit_tools: HashMap::new(),
217            model_parallel_tool_calls: HashMap::new(),
218            model_profiles: HashMap::new(),
219            tool_allowlist: Vec::new(),
220            external_tool_timeout_seconds: DEFAULT_EXTERNAL_TOOL_TIMEOUT_SECONDS,
221            command_shell: roder_api::command_shell::default_command_shell(),
222            workspace: None,
223            policy_mode: PolicyMode::Default,
224            agent_swarm_mode: false,
225            ultra_mode: false,
226            runtime_profile: RuntimeProfile::Interactive,
227            inference_router: RuntimeInferenceRouterConfig::default(),
228            speed_policy: RuntimeSpeedPolicyConfig::default(),
229            dynamic_workflows: RuntimeDynamicWorkflowConfig::default(),
230            reliability: RuntimeReliabilityConfig::default(),
231            turn_deadline_seconds: None,
232            remote_runner_destination: None,
233            team_data_dir: None,
234            roadmap_data_dir: None,
235            media_generation: crate::media_generation::RuntimeMediaGenerationConfig::default(),
236            review: crate::review::RuntimeReviewConfig::default(),
237        }
238    }
239}
240
241#[derive(Debug, Clone)]
242pub struct StartTurnRequest {
243    pub thread_id: ThreadId,
244    pub message: String,
245    pub images: Vec<InputImage>,
246    pub provider_override: Option<String>,
247    pub model_override: Option<String>,
248    pub reasoning_override: Option<String>,
249    pub workspace: String,
250    pub instructions: InstructionBundle,
251    /**
252     * Per-turn developer-authority context for this turn's InstructionBundle.
253     * Applies to every inference round of the turn, is never written to
254     * thread state. Automatic goal continuations retain it; later explicit
255     * turns receive only the context supplied by their host.
256     */
257    pub developer_context: Option<String>,
258    pub task_ledger_required: bool,
259    /**
260     * Per-turn provider service tier (OpenAI `service_tier`, e.g.
261     * `"priority"` for Fast mode). Carried to every inference round of the
262     * turn as `RuntimeHints::service_tier`; `None` inherits the thread and provider
263     * default. Providers without a tier concept ignore it.
264     */
265    pub service_tier_override: Option<String>,
266}
267
268#[derive(Debug, Clone)]
269pub struct CreateThreadRequest {
270    /// Default provider service tier for this thread.
271    pub service_tier: Option<String>,
272    pub title: Option<String>,
273    pub workspace: String,
274    pub workspace_id: Option<String>,
275    pub root_id: Option<String>,
276    pub provider: Option<String>,
277    pub model: Option<String>,
278    pub selection_mode: Option<ModelSelectionMode>,
279    /// Per-thread tool filter applied on top of the runtime allowlist. Empty = no filtering.
280    pub tool_allowlist: Vec<String>,
281    /// Host-supplied instructions added to the developer slot of every turn's inference request.
282    pub developer_instructions: Option<String>,
283    /// Host-executed tool specs advertised to the model on every turn of this thread.
284    pub external_tools: Vec<roder_api::tools::ToolSpec>,
285    /// Explicit remote-runner binding for the thread's native coding tools.
286    pub runner: Option<ThreadRunnerSelection>,
287}
288
289/**
290 * Thread-level remote-runner selection. The destination config is persisted
291 * with the thread, so secrets must reach the provider through its
292 * environment, not this config.
293 */
294#[derive(Debug, Clone)]
295pub struct ThreadRunnerSelection {
296    pub provider_id: String,
297    pub config: serde_json::Value,
298    /// Absolute path on the runner used as the thread's coding-tool workspace root.
299    pub workspace: String,
300    /**
301     * Extra absolute runner paths file reads may resolve under, beyond
302     * `workspace`. Writes and the working directory stay confined to
303     * `workspace`.
304     */
305    pub read_roots: Vec<String>,
306}
307
308#[derive(Debug, Clone, PartialEq, Eq)]
309pub struct PendingPlanExit {
310    pub thread_id: ThreadId,
311    pub turn_id: TurnId,
312    pub request_id: String,
313    pub target_mode: PolicyMode,
314    pub plan_summary: Option<String>,
315    pub next_steps: Vec<String>,
316    pub requested_at: OffsetDateTime,
317    pub expires_at: Option<OffsetDateTime>,
318}
319
320pub(crate) struct PendingToolApproval {
321    pub(crate) thread_id: ThreadId,
322    pub(crate) turn_id: TurnId,
323    pub(crate) tool_id: String,
324    pub(crate) tool_name: String,
325    pub(crate) call: roder_api::tools::ToolCall,
326    pub(crate) context: Option<ToolExecutionContext>,
327    pub(crate) tx: oneshot::Sender<bool>,
328}
329
330pub(crate) struct PendingUserInput {
331    pub(crate) thread_id: ThreadId,
332    pub(crate) turn_id: TurnId,
333    pub(crate) tx: oneshot::Sender<serde_json::Value>,
334}
335
336/// Host answer to an external tool call delivered via `tools/resolve`.
337#[derive(Debug, Clone, PartialEq, Eq)]
338pub struct ExternalToolResolution {
339    pub output: String,
340    pub is_error: bool,
341}
342
343pub(crate) struct PendingExternalToolCall {
344    pub(crate) thread_id: ThreadId,
345    pub(crate) turn_id: TurnId,
346    pub(crate) tool_id: String,
347    pub(crate) tool_name: String,
348    pub(crate) tx: oneshot::Sender<ExternalToolResolution>,
349}
350
351#[derive(Clone)]
352struct ActiveTurnHandle {
353    thread_id: ThreadId,
354    abort: AbortHandle,
355    steers: Arc<Mutex<Vec<QueuedTurnSteer>>>,
356    steer_changed: tokio::sync::watch::Sender<u64>,
357    drain: Arc<TurnDrainHandle>,
358}
359
360/// Tracks a task after it has been interrupted and removed from the interactive
361/// active-turn map. This lets users start a follow-up turn immediately while a
362/// bounded runtime shutdown can still await the original task's cleanup.
363struct TurnDrainHandle {
364    thread_id: ThreadId,
365    interrupt_requested: AtomicBool,
366    interrupt_reason: Mutex<Option<TurnLifecycleReason>>,
367    completed: AtomicBool,
368    completed_notify: Notify,
369}
370
371/// Result of a bounded runtime drain. `Clean` proves the runtime's own turn
372/// tasks have reached their cleanup paths; provider-specific child-process
373/// reaping remains an explicit provider capability.
374#[derive(Debug, Clone, PartialEq, Eq)]
375pub enum RuntimeDrainOutcome {
376    Clean {
377        interrupted_turn_ids: Vec<TurnId>,
378    },
379    DeadlineExceeded {
380        interrupted_turn_ids: Vec<TurnId>,
381        remaining_turn_ids: Vec<TurnId>,
382    },
383    /// At least one lifecycle transition initiated by this drain could not be
384    /// durably appended. The runtime still made its best cleanup attempt, but
385    /// callers must not claim a persisted terminal state as proof of recovery.
386    PersistenceFailed {
387        interrupted_turn_ids: Vec<TurnId>,
388        remaining_turn_ids: Vec<TurnId>,
389    },
390}
391
392#[derive(Clone)]
393struct QueuedTurnSteer {
394    message: UserMessage,
395    mailbox_ack: Option<MailboxDeliveryAck>,
396}
397
398#[derive(Clone)]
399struct MailboxDeliveryAck {
400    team_id: TeamId,
401    message_ids: Vec<String>,
402}
403
404#[derive(Clone)]
405struct InheritedTurnContext {
406    workspace: String,
407    instructions: InstructionBundle,
408    developer_context: Option<String>,
409}
410
411#[derive(Debug, Clone, Default, PartialEq, Eq)]
412pub struct ThreadActivity {
413    pub active_turn_id: Option<TurnId>,
414    pub active_flags: Vec<String>,
415}
416
417/// Per-thread settings persisted at thread creation and applied to every turn.
418#[derive(Debug, Clone, Default)]
419pub(crate) struct ThreadTurnOverrides {
420    pub(crate) tool_allowlist: Vec<String>,
421    pub(crate) developer_instructions: Option<String>,
422    pub(crate) external_tools: Vec<roder_api::tools::ToolSpec>,
423    pub(crate) service_tier: Option<String>,
424}
425
426#[derive(Debug, Clone, Copy, PartialEq, Eq)]
427pub(crate) enum TurnRunOutcome {
428    Completed,
429    Stopped,
430}
431
432impl PendingPlanExit {
433    pub fn new(
434        thread_id: ThreadId,
435        turn_id: TurnId,
436        request_id: String,
437        target_mode: PolicyMode,
438        plan_summary: Option<String>,
439        next_steps: Vec<String>,
440    ) -> Self {
441        let requested_at = OffsetDateTime::now_utc();
442        Self {
443            thread_id,
444            turn_id,
445            request_id,
446            target_mode,
447            plan_summary,
448            next_steps,
449            requested_at,
450            expires_at: Some(requested_at + default_plan_exit_timeout()),
451        }
452    }
453
454    pub fn is_expired(&self, now: OffsetDateTime) -> bool {
455        self.expires_at.is_some_and(|expires_at| now >= expires_at)
456    }
457}
458
459pub fn default_plan_exit_timeout() -> Duration {
460    Duration::minutes(10)
461}
462
463pub const DEFAULT_EXTERNAL_TOOL_TIMEOUT_SECONDS: u64 = 300;
464
465fn format_mailbox_messages(team: &TeamState, messages: &[TeamMailboxMessage]) -> String {
466    messages
467        .iter()
468        .map(|message| {
469            let recipient = team
470                .members
471                .iter()
472                .find(|member| member.id == message.to_member_id)
473                .map(canonical_team_member_path)
474                .unwrap_or_else(|| format!("/root/{}", message.to_member_id));
475            let sender = message
476                .from_member_id
477                .as_deref()
478                .and_then(|id| team.members.iter().find(|member| member.id == id))
479                .map(canonical_team_member_path)
480                .unwrap_or_else(|| "/root".to_string());
481            let message_type = match message.kind {
482                TeamMailboxMessageKind::Message => "MESSAGE",
483                TeamMailboxMessageKind::NewTask => "NEW_TASK",
484                TeamMailboxMessageKind::FinalAnswer => "FINAL_ANSWER",
485            };
486            format!(
487                "Message Type: {message_type}\nTask name: {recipient}\nSender: {sender}\nPayload:\n{}",
488                message.text
489            )
490        })
491        .collect::<Vec<_>>()
492        .join("\n\n")
493}
494
495fn canonical_team_member_path(member: &TeamMemberDescriptor) -> String {
496    if let Some(agent_path) = member
497        .agent_path
498        .as_deref()
499        .filter(|path| *path == "/root" || path.starts_with("/root/"))
500    {
501        return agent_path.to_string();
502    }
503    if member.role == TeamMemberRole::Lead {
504        return "/root".to_string();
505    }
506    member
507        .task_name
508        .as_deref()
509        .filter(|name| !name.trim().is_empty())
510        .map(|name| format!("/root/{}", name.trim().trim_matches('/')))
511        .unwrap_or_else(|| format!("/root/{}", member.id))
512}
513
514fn is_codex_v2_team(team: &TeamState) -> bool {
515    team.members.iter().any(|member| {
516        member
517            .model
518            .as_deref()
519            .is_some_and(|model| model_supports_reasoning_effort(model, REASONING_ULTRA))
520    })
521}
522
523fn runtime_local_team_view(
524    mut team: TeamState,
525    active_thread_ids: &std::collections::HashSet<ThreadId>,
526) -> TeamState {
527    for member in &mut team.members {
528        if member.status == TeamMemberStatus::Running
529            && !active_thread_ids.contains(&member.thread_id)
530        {
531            member.status = TeamMemberStatus::Interrupted;
532            member.current_turn_id = None;
533        }
534    }
535    team
536}
537
538type RunnerToolExecutionKey = (String, String);
539type RunnerToolExecutionMutex = Mutex<()>;
540type RunnerToolExecutionLockRegistry =
541    Mutex<HashMap<RunnerToolExecutionKey, Weak<RunnerToolExecutionMutex>>>;
542
543pub struct Runtime {
544    pub bus: EventBus,
545    pub registry: ExtensionRegistry,
546    config: RwLock<RuntimeConfig>,
547    pub(crate) execution_lease: Option<Arc<crate::execution_lease::RuntimeExecutionLease>>,
548    pending_plan_exit: RwLock<Option<PendingPlanExit>>,
549    pub(crate) pending_tool_approvals: Mutex<HashMap<String, PendingToolApproval>>,
550    pub(crate) pending_user_inputs: Mutex<HashMap<String, PendingUserInput>>,
551    pub(crate) pending_external_tool_calls: Mutex<HashMap<String, PendingExternalToolCall>>,
552    /// Serializes turn admission with shutdown quiescing. A drain takes this
553    /// gate before flipping `accepting_turns` and snapshotting active work so a
554    /// turn that observed the old flag cannot be inserted after the snapshot.
555    turn_admission: Mutex<()>,
556    active_turns: RwLock<HashMap<TurnId, ActiveTurnHandle>>,
557    turn_drains: RwLock<HashMap<TurnId, Arc<TurnDrainHandle>>>,
558    turn_usage: Mutex<HashMap<TurnId, Arc<Mutex<usage_accounting::TurnUsageAccounting>>>>,
559    thread_usage_metadata_lock: Mutex<()>,
560    /// Provider cleanup handles register synchronously through the inference
561    /// tool-executor context. They are intentionally separate from active turn
562    /// admission so an interrupted turn can remain drainable after a follow-up
563    /// turn starts on the same thread.
564    provider_turn_cleanups: std::sync::Mutex<HashMap<TurnId, Arc<dyn ProviderTurnCleanup>>>,
565    accepting_turns: AtomicBool,
566    /// Monotonic counter used by bounded drains to surface lifecycle-state
567    /// persistence failures without making an individual provider turn fail.
568    lifecycle_persistence_failures: AtomicUsize,
569    lifecycle_shutdown_drains: AtomicUsize,
570    lifecycle_clean_shutdowns: AtomicUsize,
571    lifecycle_deadline_exceeded: AtomicUsize,
572    lifecycle_persistence_failed_drains: AtomicUsize,
573    lifecycle_restart_reconciliations: AtomicUsize,
574    lifecycle_shutdown_drain_duration_ms_total: AtomicUsize,
575    provider_cleanup_confirmed: AtomicUsize,
576    provider_cleanup_timed_out: AtomicUsize,
577    provider_cleanup_unknown: AtomicUsize,
578    active_turns_changed: Notify,
579    active_turn_selections: RwLock<HashMap<TurnId, ModelSelectionMode>>,
580    compaction_templates: RwLock<HashMap<ThreadId, AgentInferenceRequest>>,
581    thread_admission_gates: Mutex<HashMap<ThreadId, Arc<Mutex<()>>>>,
582    /// Threads that have already dispatched a `SessionStart` local hook.
583    ///
584    /// `ThreadCreated` fires once, but `ThreadLoaded` fires on every read of the
585    /// thread from its store, so without this a session ran its setup hooks
586    /// several times — including after the turn had already stopped.
587    session_hook_started: RwLock<HashSet<ThreadId>>,
588    active_turn_contexts: RwLock<HashMap<TurnId, InheritedTurnContext>>,
589    /// Process-local execution boundary. Local/CLI runtimes default to true;
590    /// hosted runtime pools set this false so missing runner metadata fails
591    /// closed instead of exposing the host workspace.
592    allow_local_workspaces: AtomicBool,
593    // Spawn-time live authority retained for every reusable turn of a long-lived teammate.
594    // This stays process-local because developer_context is explicitly volatile and must never
595    // be persisted to thread state.
596    team_member_turn_contexts: Mutex<HashMap<ThreadId, InheritedTurnContext>>,
597    workspace: PathBuf,
598    pub(crate) teams: TeamManager,
599    agent_team_spawn_lock: Mutex<()>,
600    pub(crate) roadmaps: Mutex<roder_roadmap::RoadmapRuntime>,
601    /// Completed reviews, most recent last, so a later publish can resolve a
602    /// review id back to its findings. Bounded; see `review::run`.
603    pub(crate) reviews: Mutex<Vec<crate::review::ReviewRecord>>,
604    pub(crate) goals: Arc<RuntimeGoalController>,
605    context_artifacts: roder_api::artifacts::ContextArtifactStore,
606    pub(crate) thread_store: Option<Arc<dyn ThreadStore>>,
607    thread_item_cache: Mutex<ThreadItemCache>,
608    pub(crate) tool_registry: ToolRegistry,
609    pub(crate) thread_events: crate::thread_events::ThreadEvents,
610    media_generation: Arc<crate::media_generation::MediaGenerationService>,
611    pub(crate) skills: RwLock<SkillRegistry>,
612    /// Lazily-started bounded dispatch of emitted events to registry
613    /// `EventSink`s (process extensions etc.); see `event_sink_dispatch`.
614    event_sink_dispatcher: tokio::sync::OnceCell<crate::event_sink_dispatch::EventSinkDispatcher>,
615    pub(crate) compaction_hysteresis:
616        std::sync::Mutex<HashMap<ThreadId, crate::compaction_runtime::CompactionState>>,
617    /// Per-thread agent-swarm mode overrides (roadmap 104). A thread present in
618    /// this map uses its stored value; absent threads fall back to the
619    /// runtime-global `RuntimeConfig.agent_swarm_mode` default. This mirrors the
620    /// team per-member policy-mode override idiom so swarm mode is per-thread
621    /// like the other `thread/*` operations, without a separate runtime.
622    agent_swarm_modes: RwLock<HashMap<ThreadId, bool>>,
623    /// Per-thread ultra mode overrides. A thread present in this map uses its
624    /// stored value; absent threads fall back to the runtime-global
625    /// `RuntimeConfig.ultra_mode` default.
626    ultra_modes: RwLock<HashMap<ThreadId, bool>>,
627    thread_policy_modes: RwLock<HashMap<ThreadId, PolicyMode>>,
628    /**
629     * Live remote-runner sessions keyed by thread. Each per-thread mutex is
630     * held across provider initialization, making the first workspace-tool
631     * access a singleflight while allowing unrelated threads to initialize in
632     * parallel. Failed initialization is shared with current waiters, then
633     * evicted so a later tool call can retry.
634     */
635    runner_sessions: Arc<Mutex<HashMap<ThreadId, Arc<RunnerSessionSlot>>>>,
636    /**
637     * Outer tool-call locks keyed by the actual remote session, not the
638     * per-thread destination id. Hosted runtimes are tenant-scoped, so this
639     * serializes threads that share one sandbox without coupling different
640     * tenants or independent runner sessions.
641     */
642    runner_tool_execution_locks: RunnerToolExecutionLockRegistry,
643}
644
645#[derive(Clone)]
646struct CachedRunnerSession {
647    destination: RunnerDestination,
648    session: Arc<dyn RemoteRunnerSession>,
649}
650
651struct RunnerSessionSlot {
652    provider_id: String,
653    destination_id: String,
654    state: Mutex<RunnerSessionSlotState>,
655    initialized: Notify,
656}
657
658enum RunnerSessionSlotState {
659    Empty,
660    Initializing,
661    Ready(CachedRunnerSession),
662    Failed(Arc<str>),
663}
664
665impl RunnerSessionSlot {
666    fn empty(destination: &RunnerDestination) -> Self {
667        Self {
668            provider_id: destination.provider_id.clone(),
669            destination_id: destination.id.clone(),
670            state: Mutex::new(RunnerSessionSlotState::Empty),
671            initialized: Notify::new(),
672        }
673    }
674
675    fn ready(session: CachedRunnerSession) -> Self {
676        Self {
677            provider_id: session.destination.provider_id.clone(),
678            destination_id: session.destination.id.clone(),
679            state: Mutex::new(RunnerSessionSlotState::Ready(session)),
680            initialized: Notify::new(),
681        }
682    }
683
684    fn matches(&self, destination: &RunnerDestination) -> bool {
685        self.provider_id == destination.provider_id && self.destination_id == destination.id
686    }
687}
688
689impl Runtime {
690    pub fn new(registry: ExtensionRegistry, config: RuntimeConfig) -> anyhow::Result<Self> {
691        if registry.inference_engines.is_empty() {
692            anyhow::bail!("at least one inference engine must be registered");
693        }
694        validate_runtime_config_reasoning(&config)?;
695        validate_runtime_inference_router_config(&registry, &config)?;
696
697        let bus = EventBus::new(EVENT_BUS_CAPACITY);
698        let thread_store = registry
699            .thread_stores
700            .first()
701            .map(|factory| factory.create());
702        let mut tool_registry = ToolRegistry::default();
703        for contributor in &registry.tools {
704            contributor
705                .contribute(&mut tool_registry)
706                .with_context(|| format!("tool contributor {} failed", contributor.id()))?;
707        }
708        crate::agent_control_tools::contribute_agent_control_tools(&mut tool_registry)?;
709        crate::thread_events::contribute_tools(&mut tool_registry)?;
710
711        let media_generation = Arc::new(crate::media_generation::MediaGenerationService::new(
712            registry.media_generator_providers.clone(),
713            config.media_generation.clone(),
714        ));
715        tool_registry.replace(Arc::new(
716            crate::media_generation::MediaGenerateImageTool::new(media_generation.clone()),
717        ));
718
719        let team_data_dir = config.team_data_dir.clone();
720        let workspace = config
721            .workspace
722            .clone()
723            .map(PathBuf::from)
724            .unwrap_or(std::env::current_dir()?);
725        let roadmap_data_dir = config
726            .roadmap_data_dir
727            .clone()
728            .unwrap_or_else(|| workspace.join(".roder"));
729        let context_artifacts = thread_store
730            .as_ref()
731            .and_then(|store| store.context_artifact_store())
732            .or_else(|| {
733                thread_store
734                    .as_ref()
735                    .and_then(|store| store.local_thread_root())
736                    .map(FilesystemContextArtifactStore::shared_thread_scoped)
737            })
738            .unwrap_or_else(|| {
739                FilesystemContextArtifactStore::shared_legacy(default_context_artifact_dir())
740            });
741        let goals = Arc::new(RuntimeGoalController::new(
742            bus.clone(),
743            thread_store.clone(),
744        ));
745        let runtime = Self {
746            bus,
747            registry,
748            config: RwLock::new(config),
749            execution_lease: None,
750            pending_plan_exit: RwLock::new(None),
751            pending_tool_approvals: Mutex::new(HashMap::new()),
752            pending_user_inputs: Mutex::new(HashMap::new()),
753            pending_external_tool_calls: Mutex::new(HashMap::new()),
754            turn_admission: Mutex::new(()),
755            active_turns: RwLock::new(HashMap::new()),
756            turn_drains: RwLock::new(HashMap::new()),
757            turn_usage: Mutex::new(HashMap::new()),
758            thread_usage_metadata_lock: Mutex::new(()),
759            provider_turn_cleanups: std::sync::Mutex::new(HashMap::new()),
760            accepting_turns: AtomicBool::new(true),
761            lifecycle_persistence_failures: AtomicUsize::new(0),
762            lifecycle_shutdown_drains: AtomicUsize::new(0),
763            lifecycle_clean_shutdowns: AtomicUsize::new(0),
764            lifecycle_deadline_exceeded: AtomicUsize::new(0),
765            lifecycle_persistence_failed_drains: AtomicUsize::new(0),
766            lifecycle_restart_reconciliations: AtomicUsize::new(0),
767            lifecycle_shutdown_drain_duration_ms_total: AtomicUsize::new(0),
768            provider_cleanup_confirmed: AtomicUsize::new(0),
769            provider_cleanup_timed_out: AtomicUsize::new(0),
770            provider_cleanup_unknown: AtomicUsize::new(0),
771            active_turns_changed: Notify::new(),
772            active_turn_selections: RwLock::new(HashMap::new()),
773            compaction_templates: RwLock::new(HashMap::new()),
774            thread_admission_gates: Mutex::new(HashMap::new()),
775            session_hook_started: RwLock::new(HashSet::new()),
776            active_turn_contexts: RwLock::new(HashMap::new()),
777            allow_local_workspaces: AtomicBool::new(true),
778            team_member_turn_contexts: Mutex::new(HashMap::new()),
779            workspace: workspace.clone(),
780            teams: TeamManager::new(
781                team_data_dir.unwrap_or_else(crate::teams::default_team_data_dir),
782            ),
783            agent_team_spawn_lock: Mutex::new(()),
784            roadmaps: Mutex::new(roder_roadmap::RoadmapRuntime::new(
785                workspace,
786                roadmap_data_dir,
787            )),
788            reviews: Mutex::new(Vec::new()),
789            goals,
790            context_artifacts,
791            thread_store,
792            thread_item_cache: Mutex::new(ThreadItemCache::default()),
793            tool_registry,
794            thread_events: crate::thread_events::ThreadEvents::default(),
795            media_generation,
796            skills: RwLock::new(SkillRegistry::load(SkillRegistryOptions::new(
797                PathBuf::new(),
798            ))),
799            event_sink_dispatcher: tokio::sync::OnceCell::new(),
800            compaction_hysteresis: crate::compaction_runtime::compaction_hysteresis_state(),
801            agent_swarm_modes: RwLock::new(HashMap::new()),
802            ultra_modes: RwLock::new(HashMap::new()),
803            thread_policy_modes: RwLock::new(HashMap::new()),
804            runner_sessions: Arc::new(Mutex::new(HashMap::new())),
805            runner_tool_execution_locks: Mutex::new(HashMap::new()),
806        };
807        runtime.bus.emit(RoderEvent::RuntimeStarted(RuntimeStarted {
808            timestamp: OffsetDateTime::now_utc(),
809        }));
810        for manifest in &runtime.registry.manifests {
811            runtime
812                .bus
813                .emit(RoderEvent::ExtensionRegistered(ExtensionRegistered {
814                    extension_id: manifest.id.clone(),
815                    timestamp: OffsetDateTime::now_utc(),
816                }));
817        }
818        Ok(runtime)
819    }
820
821    pub fn from_engine(engine: Arc<dyn InferenceEngine>) -> anyhow::Result<Self> {
822        let mut builder = roder_api::extension::ExtensionRegistryBuilder::new();
823        builder.inference_engine(engine);
824        Self::new(builder.build()?, RuntimeConfig::default())
825    }
826
827    pub fn fake() -> anyhow::Result<Self> {
828        Self::from_engine(Arc::new(FakeInferenceEngine))
829    }
830
831    pub(crate) fn accepting_thread_events(&self) -> bool {
832        self.accepting_turns.load(Ordering::Acquire)
833    }
834
835    pub fn subscribe_events(&self) -> tokio::sync::broadcast::Receiver<EventEnvelope> {
836        self.bus.subscribe()
837    }
838
839    /// Returns process-local, redacted lifecycle health counters. The snapshot
840    /// intentionally has fixed fields and contains no provider, command,
841    /// process, thread, or turn identifiers.
842    pub fn lifecycle_metrics(&self) -> LifecycleMetricsSnapshot {
843        LifecycleMetricsSnapshot {
844            shutdown_drain_count: self.lifecycle_shutdown_drains.load(Ordering::Acquire) as u64,
845            clean_shutdown_count: self.lifecycle_clean_shutdowns.load(Ordering::Acquire) as u64,
846            deadline_exceeded_count: self.lifecycle_deadline_exceeded.load(Ordering::Acquire)
847                as u64,
848            persistence_failed_count: self
849                .lifecycle_persistence_failed_drains
850                .load(Ordering::Acquire) as u64,
851            restart_reconciliation_count: self
852                .lifecycle_restart_reconciliations
853                .load(Ordering::Acquire) as u64,
854            lifecycle_persistence_failure_count: self
855                .lifecycle_persistence_failures
856                .load(Ordering::Acquire) as u64,
857            shutdown_drain_duration_ms_total: self
858                .lifecycle_shutdown_drain_duration_ms_total
859                .load(Ordering::Acquire) as u64,
860            provider_cleanup_confirmed_count: self
861                .provider_cleanup_confirmed
862                .load(Ordering::Acquire) as u64,
863            provider_cleanup_timed_out_count: self
864                .provider_cleanup_timed_out
865                .load(Ordering::Acquire) as u64,
866            provider_cleanup_unknown_count: self.provider_cleanup_unknown.load(Ordering::Acquire)
867                as u64,
868        }
869    }
870
871    async fn persist_turn_lifecycle_record(&self, record: &TurnLifecycleRecord) -> bool {
872        let Some(store) = &self.thread_store else {
873            return true;
874        };
875        let state = match record.extension_state() {
876            Ok(state) => state,
877            Err(_) => {
878                self.lifecycle_persistence_failures
879                    .fetch_add(1, Ordering::AcqRel);
880                return false;
881            }
882        };
883        // Lifecycle diagnostics must not turn a provider result into a failed
884        // turn merely because a third-party store lacks extension-state support.
885        // JSONL/Postgres stores persist this append-only record atomically at
886        // their own storage boundary.
887        if store
888            .append_extension_state(&record.thread_id, &state)
889            .await
890            .is_err()
891        {
892            self.lifecycle_persistence_failures
893                .fetch_add(1, Ordering::AcqRel);
894            return false;
895        }
896        true
897    }
898
899    async fn record_turn_lifecycle(
900        &self,
901        thread_id: ThreadId,
902        turn_id: TurnId,
903        state: TurnLifecycleState,
904        cleanup: TurnCleanupState,
905        reason: Option<TurnLifecycleReason>,
906    ) -> TurnLifecycleRecord {
907        self.record_turn_lifecycle_with_ownership(
908            thread_id,
909            turn_id,
910            state,
911            cleanup,
912            reason,
913            TurnCleanupOwnership::RuntimeTaskOnly,
914        )
915        .await
916    }
917
918    async fn record_turn_lifecycle_with_ownership(
919        &self,
920        thread_id: ThreadId,
921        turn_id: TurnId,
922        state: TurnLifecycleState,
923        cleanup: TurnCleanupState,
924        reason: Option<TurnLifecycleReason>,
925        ownership: TurnCleanupOwnership,
926    ) -> TurnLifecycleRecord {
927        let record = TurnLifecycleRecord::new(
928            thread_id,
929            turn_id,
930            state,
931            cleanup,
932            reason,
933            OffsetDateTime::now_utc(),
934        )
935        .with_ownership(ownership);
936        let _ = self.persist_turn_lifecycle_record(&record).await;
937        self.emit(RoderEvent::TurnLifecycleUpdated(record.clone()))
938            .await;
939        record
940    }
941
942    pub(crate) fn register_provider_turn_cleanup(
943        &self,
944        turn_id: &TurnId,
945        cleanup: Arc<dyn ProviderTurnCleanup>,
946    ) {
947        if let Ok(mut cleanups) = self.provider_turn_cleanups.lock() {
948            cleanups.insert(turn_id.clone(), cleanup);
949        }
950    }
951
952    fn provider_cleanup_ownership(&self, turn_id: &TurnId) -> TurnCleanupOwnership {
953        self.provider_turn_cleanups
954            .lock()
955            .ok()
956            .and_then(|cleanups| cleanups.get(turn_id).cloned())
957            .map(|cleanup| cleanup.ownership())
958            .unwrap_or(TurnCleanupOwnership::RuntimeTaskOnly)
959    }
960
961    async fn await_provider_turn_cleanup(
962        &self,
963        turn_id: &TurnId,
964    ) -> (TurnCleanupState, TurnCleanupOwnership) {
965        let cleanup = self
966            .provider_turn_cleanups
967            .lock()
968            .ok()
969            .and_then(|mut cleanups| cleanups.remove(turn_id));
970        let Some(cleanup) = cleanup else {
971            return (
972                TurnCleanupState::Unknown,
973                TurnCleanupOwnership::RuntimeTaskOnly,
974            );
975        };
976        let ownership = cleanup.ownership();
977        match tokio::time::timeout(PROVIDER_CLEANUP_TIMEOUT, cleanup.wait_for_cleanup()).await {
978            Ok(Ok(())) => {
979                self.provider_cleanup_confirmed
980                    .fetch_add(1, Ordering::AcqRel);
981                (
982                    TurnCleanupState::Completed,
983                    TurnCleanupOwnership::ProviderCleanupConfirmed,
984                )
985            }
986            Ok(Err(_)) => {
987                self.provider_cleanup_unknown.fetch_add(1, Ordering::AcqRel);
988                (TurnCleanupState::Unknown, ownership)
989            }
990            Err(_) => {
991                self.provider_cleanup_timed_out
992                    .fetch_add(1, Ordering::AcqRel);
993                (TurnCleanupState::TimedOut, ownership)
994            }
995        }
996    }
997
998    fn record_lifecycle_drain_outcome(
999        &self,
1000        started_at: tokio::time::Instant,
1001        outcome: &RuntimeDrainOutcome,
1002    ) {
1003        self.lifecycle_shutdown_drains
1004            .fetch_add(1, Ordering::AcqRel);
1005        self.lifecycle_shutdown_drain_duration_ms_total.fetch_add(
1006            started_at.elapsed().as_millis().min(usize::MAX as u128) as usize,
1007            Ordering::AcqRel,
1008        );
1009        match outcome {
1010            RuntimeDrainOutcome::Clean { .. } => {
1011                self.lifecycle_clean_shutdowns
1012                    .fetch_add(1, Ordering::AcqRel);
1013            }
1014            RuntimeDrainOutcome::DeadlineExceeded { .. } => {
1015                self.lifecycle_deadline_exceeded
1016                    .fetch_add(1, Ordering::AcqRel);
1017            }
1018            RuntimeDrainOutcome::PersistenceFailed { .. } => {
1019                self.lifecycle_persistence_failed_drains
1020                    .fetch_add(1, Ordering::AcqRel);
1021            }
1022        }
1023    }
1024
1025    fn lifecycle_record_for_event(event: &RoderEvent) -> Option<TurnLifecycleRecord> {
1026        match event {
1027            RoderEvent::TurnStarted(event) => Some(TurnLifecycleRecord::new(
1028                event.thread_id.clone(),
1029                event.turn_id.clone(),
1030                TurnLifecycleState::Running,
1031                TurnCleanupState::NotRequested,
1032                None,
1033                event.timestamp,
1034            )),
1035            // `run_turn` persists completed turns after it has awaited a
1036            // registered provider cleanup handle. Do not let the generic event
1037            // projection overwrite that acknowledgement with an unproven state.
1038            RoderEvent::TurnCompleted(_) => None,
1039            // Some failure paths already have an event before the turn wrapper
1040            // reaches its cleanup acknowledgement. Preserve a durable fallback
1041            // reason here; the wrapper appends a later record with the more
1042            // precise cleanup outcome when a provider registered one.
1043            RoderEvent::TurnFailed(event) => Some(TurnLifecycleRecord::new(
1044                event.thread_id.clone(),
1045                event.turn_id.clone(),
1046                TurnLifecycleState::Failed,
1047                TurnCleanupState::Unknown,
1048                Some(if event.error_kind.as_deref() == Some("deadline_timeout") {
1049                    TurnLifecycleReason::DeadlineExceeded
1050                } else {
1051                    TurnLifecycleReason::ProviderFailure
1052                }),
1053                event.timestamp,
1054            )),
1055            RoderEvent::TurnDeadlineExceeded(_) | RoderEvent::TurnInterrupted(_) => None,
1056            RoderEvent::TurnLifecycleUpdated(_) => None,
1057            _ => None,
1058        }
1059    }
1060
1061    pub fn registry(&self) -> &ExtensionRegistry {
1062        &self.registry
1063    }
1064
1065    pub fn media_generation(&self) -> Arc<crate::media_generation::MediaGenerationService> {
1066        self.media_generation.clone()
1067    }
1068
1069    pub fn context_artifacts(&self) -> roder_api::artifacts::ContextArtifactStore {
1070        self.context_artifacts.clone()
1071    }
1072
1073    pub async fn execute_workflow_tool(
1074        &self,
1075        thread_id: ThreadId,
1076        tool_name: &str,
1077        arguments: serde_json::Value,
1078    ) -> anyhow::Result<ToolResult> {
1079        let Some(executor) = self.tool_registry.get(tool_name) else {
1080            anyhow::bail!("tool not found: {tool_name}");
1081        };
1082        let tool_call = ToolCall {
1083            id: format!("slash-{tool_name}"),
1084            name: tool_name.to_string(),
1085            raw_arguments: serde_json::to_string(&arguments)?,
1086            arguments,
1087            thread_id: thread_id.clone(),
1088            turn_id: "slash-command".to_string(),
1089        };
1090        let runtime_config = self.status().await;
1091        let ctx = self.tool_execution_context(
1092            thread_id,
1093            "slash-command".to_string(),
1094            runtime_config.policy_mode,
1095            runtime_config.workspace.as_deref(),
1096            Some(&runtime_config.command_shell),
1097        );
1098        executor.execute(ctx, tool_call).await
1099    }
1100
1101    pub(crate) fn tool_execution_context(
1102        &self,
1103        thread_id: ThreadId,
1104        turn_id: TurnId,
1105        mode: PolicyMode,
1106        workspace: Option<&str>,
1107        command_shell: Option<&str>,
1108    ) -> ToolExecutionContext {
1109        let mut ctx = ToolExecutionContext::new(thread_id, turn_id, mode)
1110            .with_command_shell(command_shell.unwrap_or_default())
1111            .with_process_runner(Arc::new(LocalProcessRunner))
1112            .with_context_artifacts(self.context_artifacts.backend())
1113            .with_goal_controller(self.goals.clone())
1114            .with_subagent_trace_sink(Arc::new(RuntimeSubagentTraceSink::new(
1115                self.bus.clone(),
1116                self.thread_store.clone(),
1117            )))
1118            .with_swarm_progress_sink(Arc::new(RuntimeAgentSwarmProgressSink::new(
1119                self.bus.clone(),
1120                self.thread_store.clone(),
1121            )));
1122        if let Some(workspace) = workspace {
1123            ctx = ctx.with_workspace_handle(Arc::new(ScopedFilesystem::new(workspace)));
1124        }
1125        ctx
1126    }
1127
1128    pub async fn status(&self) -> RuntimeConfig {
1129        self.config.read().await.clone()
1130    }
1131
1132    pub async fn set_skills(&self, skills: SkillRegistry) {
1133        *self.skills.write().await = skills;
1134    }
1135
1136    pub async fn skills_snapshot(&self) -> SkillRegistry {
1137        self.skills.read().await.clone()
1138    }
1139
1140    pub fn workspace(&self) -> PathBuf {
1141        self.workspace.clone()
1142    }
1143
1144    /// Controls whether native workspace tools may execute without an
1145    /// explicit remote-runner binding. Hosted runtimes disable this after
1146    /// construction; ordinary local runtimes retain the compatible default.
1147    pub fn set_allow_local_workspaces(&self, allowed: bool) {
1148        self.allow_local_workspaces
1149            .store(allowed, Ordering::Relaxed);
1150    }
1151
1152    pub fn allows_local_workspaces(&self) -> bool {
1153        self.allow_local_workspaces.load(Ordering::Relaxed)
1154    }
1155
1156    pub async fn set_remote_runner_destination(&self, destination: Option<RunnerDestination>) {
1157        let lifecycle = destination.as_ref().map(|destination| RunnerLifecycle {
1158            destination_id: destination.id.clone(),
1159            provider_id: destination.provider_id.clone(),
1160            state: "configured".to_string(),
1161            session_id: None,
1162            timestamp: OffsetDateTime::now_utc(),
1163        });
1164        self.config.write().await.remote_runner_destination = destination;
1165        if let Some(lifecycle) = lifecycle {
1166            self.emit(RoderEvent::RunnerLifecycle(lifecycle)).await;
1167        } else {
1168            self.emit(RoderEvent::RunnerLifecycle(RunnerLifecycle {
1169                destination_id: "local".to_string(),
1170                provider_id: "local".to_string(),
1171                state: "local_fallback".to_string(),
1172                session_id: None,
1173                timestamp: OffsetDateTime::now_utc(),
1174            }))
1175            .await;
1176        }
1177    }
1178
1179    pub async fn set_file_backed_dynamic_context(&self, enabled: bool) -> RuntimeConfig {
1180        let mut cfg = self.config.write().await;
1181        cfg.file_backed_dynamic_context = enabled;
1182        cfg.clone()
1183    }
1184
1185    pub async fn set_command_shell(&self, shell: String) -> RuntimeConfig {
1186        let mut cfg = self.config.write().await;
1187        cfg.command_shell = shell;
1188        cfg.clone()
1189    }
1190
1191    pub async fn pending_plan_exit(&self) -> Option<PendingPlanExit> {
1192        let mut pending = self.pending_plan_exit.write().await;
1193        let current = pending.clone()?;
1194        if !current.is_expired(OffsetDateTime::now_utc()) {
1195            return Some(current);
1196        }
1197        *pending = None;
1198        drop(pending);
1199        let mode = self
1200            .effective_policy_mode_for_thread(&current.thread_id)
1201            .await;
1202        self.emit_plan_exit_resolved(&current, false, mode).await;
1203        None
1204    }
1205
1206    pub async fn set_policy_mode(
1207        &self,
1208        mode: PolicyMode,
1209        reason: Option<String>,
1210    ) -> anyhow::Result<RuntimeConfig> {
1211        let mut cfg = self.config.write().await;
1212        let previous_mode = cfg.policy_mode;
1213        cfg.policy_mode = mode;
1214        let next = cfg.clone();
1215        drop(cfg);
1216        self.emit(RoderEvent::PolicyModeChanged(PolicyModeChanged {
1217            thread_id: "runtime".to_string(),
1218            turn_id: None,
1219            previous_mode,
1220            new_mode: mode,
1221            reason,
1222            timestamp: OffsetDateTime::now_utc(),
1223        }))
1224        .await;
1225        self.auto_resolve_pending_tool_approvals().await;
1226        Ok(next)
1227    }
1228
1229    /// Toggle agent-swarm mode for the runtime (roadmap 104). When enabled, the
1230    /// swarm reminder is injected into each turn's developer instructions so
1231    /// every app-server/SDK client gets it, not only the TUI.
1232    pub async fn set_agent_swarm_mode(
1233        &self,
1234        enabled: bool,
1235        trigger: roder_api::subagents::AgentSwarmModeTrigger,
1236    ) -> anyhow::Result<RuntimeConfig> {
1237        let mut cfg = self.config.write().await;
1238        cfg.agent_swarm_mode = enabled;
1239        let next = cfg.clone();
1240        drop(cfg);
1241        self.emit(RoderEvent::AgentSwarmModeChanged(
1242            roder_api::subagents::AgentSwarmModeChanged {
1243                thread_id: "runtime".to_string(),
1244                turn_id: None,
1245                enabled,
1246                trigger,
1247                timestamp: OffsetDateTime::now_utc(),
1248            },
1249        ))
1250        .await;
1251        Ok(next)
1252    }
1253
1254    /// Toggle agent-swarm mode for a single thread (roadmap 104). The override is
1255    /// stored per thread so toggling swarm mode on one thread does not leak the
1256    /// reminder into other threads sharing the runtime; absent threads fall back
1257    /// to the runtime-global default. The emitted `AgentSwarmModeChanged` event
1258    /// carries the real `thread_id`.
1259    pub async fn set_agent_swarm_mode_for_thread(
1260        &self,
1261        thread_id: &str,
1262        enabled: bool,
1263        trigger: roder_api::subagents::AgentSwarmModeTrigger,
1264    ) -> bool {
1265        {
1266            let mut modes = self.agent_swarm_modes.write().await;
1267            modes.insert(thread_id.to_string(), enabled);
1268        }
1269        self.emit(RoderEvent::AgentSwarmModeChanged(
1270            roder_api::subagents::AgentSwarmModeChanged {
1271                thread_id: thread_id.to_string(),
1272                turn_id: None,
1273                enabled,
1274                trigger,
1275                timestamp: OffsetDateTime::now_utc(),
1276            },
1277        ))
1278        .await;
1279        enabled
1280    }
1281
1282    /// Resolve whether agent-swarm mode is active for `thread_id`: the per-thread
1283    /// override when present, otherwise the runtime-global default. The turn loop
1284    /// uses this to decide whether to inject the swarm reminder.
1285    pub async fn effective_agent_swarm_mode_for_thread(&self, thread_id: &str) -> bool {
1286        if let Some(enabled) = self.agent_swarm_modes.read().await.get(thread_id).copied() {
1287            return enabled;
1288        }
1289        self.status().await.agent_swarm_mode
1290    }
1291
1292    /// Toggle ultra mode for the runtime. When enabled, proactive multi-agent
1293    /// policy is injected into each turn's developer instructions for any
1294    /// model, so every app-server/SDK client gets it (not only the TUI).
1295    pub async fn set_ultra_mode(
1296        &self,
1297        enabled: bool,
1298        trigger: roder_api::subagents::UltraModeTrigger,
1299    ) -> anyhow::Result<RuntimeConfig> {
1300        let mut cfg = self.config.write().await;
1301        cfg.ultra_mode = enabled;
1302        let next = cfg.clone();
1303        drop(cfg);
1304        self.emit(RoderEvent::UltraModeChanged(
1305            roder_api::subagents::UltraModeChanged {
1306                thread_id: "runtime".to_string(),
1307                turn_id: None,
1308                enabled,
1309                trigger,
1310                timestamp: OffsetDateTime::now_utc(),
1311            },
1312        ))
1313        .await;
1314        Ok(next)
1315    }
1316
1317    /// Toggle ultra mode for a single thread. The override is stored per thread
1318    /// so toggling on one thread does not leak proactive multi-agent policy into
1319    /// other threads sharing the runtime; absent threads fall back to the
1320    /// runtime-global default.
1321    pub async fn set_ultra_mode_for_thread(
1322        &self,
1323        thread_id: &str,
1324        enabled: bool,
1325        trigger: roder_api::subagents::UltraModeTrigger,
1326    ) -> bool {
1327        {
1328            let mut modes = self.ultra_modes.write().await;
1329            modes.insert(thread_id.to_string(), enabled);
1330        }
1331        self.emit(RoderEvent::UltraModeChanged(
1332            roder_api::subagents::UltraModeChanged {
1333                thread_id: thread_id.to_string(),
1334                turn_id: None,
1335                enabled,
1336                trigger,
1337                timestamp: OffsetDateTime::now_utc(),
1338            },
1339        ))
1340        .await;
1341        enabled
1342    }
1343
1344    /// Resolve whether ultra mode is active for `thread_id`: the per-thread
1345    /// override when present, otherwise the runtime-global default. The turn
1346    /// loop uses this (together with Sol/Terra Ultra effort) to decide whether
1347    /// to inject proactive multi-agent policy.
1348    pub async fn effective_ultra_mode_for_thread(&self, thread_id: &str) -> bool {
1349        if let Some(enabled) = self.ultra_modes.read().await.get(thread_id).copied() {
1350            return enabled;
1351        }
1352        self.status().await.ultra_mode
1353    }
1354
1355    pub async fn set_hosted_web_search(
1356        &self,
1357        mode: HostedWebSearchMode,
1358    ) -> anyhow::Result<RuntimeConfig> {
1359        let mut cfg = self.config.write().await;
1360        cfg.hosted_web_search = HostedWebSearchConfig { mode };
1361        Ok(cfg.clone())
1362    }
1363
1364    pub async fn record_pending_plan_exit(&self, pending: PendingPlanExit) {
1365        *self.pending_plan_exit.write().await = Some(pending.clone());
1366        self.emit(RoderEvent::PolicyExitPlanRequested(
1367            PolicyExitPlanRequested {
1368                thread_id: pending.thread_id,
1369                turn_id: pending.turn_id,
1370                request_id: pending.request_id,
1371                target_mode: pending.target_mode,
1372                plan_summary: pending.plan_summary,
1373                next_steps: pending.next_steps,
1374                timestamp: OffsetDateTime::now_utc(),
1375            },
1376        ))
1377        .await;
1378    }
1379
1380    pub async fn resolve_pending_plan_exit(
1381        &self,
1382        request_id: &str,
1383        approved: bool,
1384    ) -> anyhow::Result<Option<PendingPlanExit>> {
1385        let mut pending = self.pending_plan_exit.write().await;
1386        let Some(current) = pending.clone() else {
1387            return Ok(None);
1388        };
1389        if current.request_id != request_id {
1390            anyhow::bail!("pending plan exit request {request_id:?} was not found");
1391        }
1392        *pending = None;
1393        drop(pending);
1394
1395        let approved = approved && !current.is_expired(OffsetDateTime::now_utc());
1396        let resolved_mode = if approved {
1397            let previous_mode = self
1398                .effective_policy_mode_for_thread(&current.thread_id)
1399                .await;
1400            if self.thread_policy_mode(&current.thread_id).await?.is_some() {
1401                self.set_thread_policy_mode(&current.thread_id, current.target_mode)
1402                    .await?;
1403            } else {
1404                self.config.write().await.policy_mode = current.target_mode;
1405                self.emit(RoderEvent::PolicyModeChanged(PolicyModeChanged {
1406                    thread_id: current.thread_id.clone(),
1407                    turn_id: Some(current.turn_id.clone()),
1408                    previous_mode,
1409                    new_mode: current.target_mode,
1410                    reason: Some("approved plan exit".to_string()),
1411                    timestamp: OffsetDateTime::now_utc(),
1412                }))
1413                .await;
1414                self.auto_resolve_pending_tool_approvals().await;
1415            }
1416            self.effective_policy_mode_for_thread(&current.thread_id)
1417                .await
1418        } else {
1419            self.effective_policy_mode_for_thread(&current.thread_id)
1420                .await
1421        };
1422        self.emit_plan_exit_resolved(&current, approved, resolved_mode)
1423            .await;
1424        Ok(Some(current))
1425    }
1426
1427    pub async fn resolve_tool_approval(
1428        &self,
1429        approval_id: &str,
1430        approved: bool,
1431    ) -> anyhow::Result<bool> {
1432        let pending = self.pending_tool_approvals.lock().await.remove(approval_id);
1433        let Some(pending) = pending else {
1434            return Ok(false);
1435        };
1436        self.emit(RoderEvent::ApprovalResolved(ApprovalResolved {
1437            thread_id: pending.thread_id,
1438            turn_id: pending.turn_id,
1439            approval_id: approval_id.to_string(),
1440            tool_id: pending.tool_id,
1441            tool_name: pending.tool_name,
1442            approved,
1443            timestamp: OffsetDateTime::now_utc(),
1444        }))
1445        .await;
1446        let _ = pending.tx.send(approved);
1447        Ok(true)
1448    }
1449
1450    /// Completes a pending host-executed tool call (`tools/resolve`). Returns false when the
1451    /// request id is unknown, already resolved, timed out, or cancelled by a turn interrupt.
1452    pub async fn resolve_external_tool_call(
1453        &self,
1454        request_id: &str,
1455        resolution: ExternalToolResolution,
1456    ) -> anyhow::Result<bool> {
1457        let mut calls = self.pending_external_tool_calls.lock().await;
1458        let Some(pending) = calls.get(request_id) else {
1459            return Ok(false);
1460        };
1461        self.emit_external_execution(RoderEvent::ExternalToolCallResolved(
1462            ExternalToolCallResolved {
1463                thread_id: pending.thread_id.clone(),
1464                turn_id: pending.turn_id.clone(),
1465                request_id: request_id.to_string(),
1466                tool_id: pending.tool_id.clone(),
1467                tool_name: pending.tool_name.clone(),
1468                outcome: ExternalToolCallOutcome::Resolved,
1469                is_error: resolution.is_error,
1470                timestamp: OffsetDateTime::now_utc(),
1471            },
1472        ))
1473        .await?;
1474        let Some(pending) = calls.remove(request_id) else {
1475            return Ok(false);
1476        };
1477        drop(calls);
1478        let _ = pending.tx.send(resolution);
1479        Ok(true)
1480    }
1481
1482    /// Drops every pending external tool call for the turn and reports them as cancelled so an
1483    /// interrupt does not leak requests waiting on `tools/resolve`.
1484    async fn cancel_pending_external_tool_calls_for_turn(&self, turn_id: &TurnId) {
1485        let cancelled = {
1486            let mut pending = self.pending_external_tool_calls.lock().await;
1487            let request_ids = pending
1488                .iter()
1489                .filter(|(_, call)| &call.turn_id == turn_id)
1490                .map(|(request_id, _)| request_id.clone())
1491                .collect::<Vec<_>>();
1492            request_ids
1493                .into_iter()
1494                .filter_map(|request_id| pending.remove(&request_id).map(|call| (request_id, call)))
1495                .collect::<Vec<_>>()
1496        };
1497        for (request_id, call) in cancelled {
1498            self.emit(RoderEvent::ExternalToolCallResolved(
1499                ExternalToolCallResolved {
1500                    thread_id: call.thread_id,
1501                    turn_id: call.turn_id,
1502                    request_id,
1503                    tool_id: call.tool_id,
1504                    tool_name: call.tool_name,
1505                    outcome: ExternalToolCallOutcome::Cancelled,
1506                    is_error: true,
1507                    timestamp: OffsetDateTime::now_utc(),
1508                },
1509            ))
1510            .await;
1511        }
1512    }
1513
1514    pub async fn resolve_user_input(
1515        &self,
1516        request_id: &str,
1517        answers: serde_json::Value,
1518    ) -> anyhow::Result<bool> {
1519        let pending = self.pending_user_inputs.lock().await.remove(request_id);
1520        let Some(pending) = pending else {
1521            return Ok(false);
1522        };
1523        self.emit(RoderEvent::UserInputResolved(UserInputResolved {
1524            thread_id: pending.thread_id,
1525            turn_id: pending.turn_id,
1526            request_id: request_id.to_string(),
1527            answers: answers.clone(),
1528            timestamp: OffsetDateTime::now_utc(),
1529        }))
1530        .await;
1531        let _ = pending.tx.send(answers);
1532        Ok(true)
1533    }
1534
1535    async fn emit_plan_exit_resolved(
1536        &self,
1537        current: &PendingPlanExit,
1538        approved: bool,
1539        resolved_mode: PolicyMode,
1540    ) {
1541        self.emit(RoderEvent::PolicyExitPlanResolved(PolicyExitPlanResolved {
1542            thread_id: current.thread_id.clone(),
1543            turn_id: current.turn_id.clone(),
1544            request_id: current.request_id.clone(),
1545            approved,
1546            target_mode: current.target_mode,
1547            resolved_mode,
1548            timestamp: OffsetDateTime::now_utc(),
1549        }))
1550        .await;
1551    }
1552
1553    pub async fn select_provider(
1554        &self,
1555        provider: String,
1556        model: Option<String>,
1557        reasoning: Option<String>,
1558    ) -> anyhow::Result<RuntimeConfig> {
1559        let next = self
1560            .preview_provider_selection(provider, model, reasoning)
1561            .await?;
1562        let mut cfg = self.config.write().await;
1563        *cfg = next;
1564        Ok(cfg.clone())
1565    }
1566
1567    pub async fn preview_provider_selection(
1568        &self,
1569        provider: String,
1570        model: Option<String>,
1571        reasoning: Option<String>,
1572    ) -> anyhow::Result<RuntimeConfig> {
1573        self.engine_for(&provider)?;
1574        let mut cfg = self.config.read().await.clone();
1575        cfg.default_provider = provider;
1576        if let Some(model) = model {
1577            cfg.default_model = model;
1578        }
1579        if let Some(reasoning) = reasoning {
1580            if reasoning == REASONING_NONE
1581                && !model_supports_reasoning(&cfg.default_model, &reasoning)
1582            {
1583                return Ok(cfg.clone());
1584            }
1585            validate_reasoning_effort(&cfg.default_model, &reasoning)?;
1586            cfg.reasoning = Some(reasoning);
1587        }
1588        Ok(cfg)
1589    }
1590
1591    pub async fn effective_reasoning(&self) -> String {
1592        let cfg = self.config.read().await;
1593        effective_reasoning_for_model(&cfg, &cfg.default_model)
1594    }
1595
1596    pub fn effective_reasoning_for_config(cfg: &RuntimeConfig) -> String {
1597        effective_reasoning_for_model(cfg, &cfg.default_model)
1598    }
1599
1600    pub async fn set_dynamic_workflow_effort(
1601        &self,
1602        effort_profile: DynamicWorkflowEffortProfile,
1603    ) -> RuntimeConfig {
1604        let mut cfg = self.config.write().await;
1605        cfg.dynamic_workflows.effort_profile = effort_profile;
1606        cfg.clone()
1607    }
1608
1609    pub async fn dynamic_workflow_trigger_decision(
1610        &self,
1611        message: &str,
1612    ) -> WorkflowTriggerDecision {
1613        let cfg = self.config.read().await;
1614        classify_workflow_trigger(message, &cfg.dynamic_workflows)
1615    }
1616
1617    pub async fn create_thread(&self, title: Option<String>) -> anyhow::Result<ThreadMetadata> {
1618        self.create_thread_with(CreateThreadRequest {
1619            title,
1620            workspace: self.workspace.display().to_string(),
1621            workspace_id: None,
1622            root_id: None,
1623            provider: None,
1624            model: None,
1625            selection_mode: None,
1626            tool_allowlist: Vec::new(),
1627            service_tier: None,
1628            developer_instructions: None,
1629            external_tools: Vec::new(),
1630            runner: None,
1631        })
1632        .await
1633    }
1634
1635    /**
1636     * Resolves an explicit thread-runner selection into a persisted binding.
1637     * Fails fast at thread creation when the provider is missing or rejects
1638     * the destination, instead of surfacing the error on the first tool call.
1639     */
1640    async fn resolve_thread_runner_binding(
1641        &self,
1642        thread_id: &str,
1643        selection: ThreadRunnerSelection,
1644    ) -> anyhow::Result<ThreadRunnerBinding> {
1645        let provider = self
1646            .registry
1647            .remote_runner_providers
1648            .iter()
1649            .find(|provider| provider.id() == selection.provider_id)
1650            .cloned()
1651            .ok_or_else(|| {
1652                anyhow::anyhow!(
1653                    "remote runner provider {:?} is not installed",
1654                    selection.provider_id
1655                )
1656            })?;
1657        let workspace = selection.workspace.trim();
1658        anyhow::ensure!(
1659            std::path::Path::new(workspace).is_absolute(),
1660            "runner workspace must be an absolute path on the runner: {workspace:?}"
1661        );
1662        let mut read_roots = Vec::with_capacity(selection.read_roots.len());
1663        for read_root in &selection.read_roots {
1664            let trimmed = read_root.trim();
1665            anyhow::ensure!(
1666                std::path::Path::new(trimmed).is_absolute(),
1667                "runner read root must be an absolute path on the runner: {trimmed:?}"
1668            );
1669            read_roots.push(PathBuf::from(trimmed));
1670        }
1671        let destination = RunnerDestination {
1672            id: format!("thread-{thread_id}"),
1673            provider_id: selection.provider_id,
1674            config: selection.config,
1675            default_manifest: roder_api::remote_runner::RunnerManifest::default(),
1676        };
1677        provider.validate_destination(&destination).await?;
1678        Ok(ThreadRunnerBinding {
1679            destination,
1680            workspace: PathBuf::from(workspace),
1681            read_roots,
1682        })
1683    }
1684
1685    /**
1686     * Build a per-thread binding from a runtime-level destination (selected via
1687     * the TUI runner picker or config `default_destination`) when the
1688     * destination's provider advertises a default workspace. Returns `None` when
1689     * there is no destination or the provider opts out (no default workspace),
1690     * preserving the legacy behavior where such threads keep local tools.
1691     */
1692    async fn synthesize_runtime_runner_binding(
1693        &self,
1694        thread_id: &str,
1695        destination: Option<RunnerDestination>,
1696    ) -> anyhow::Result<Option<ThreadRunnerBinding>> {
1697        let Some(destination) = destination else {
1698            return Ok(None);
1699        };
1700        let Some(provider) = self
1701            .registry
1702            .remote_runner_providers
1703            .iter()
1704            .find(|provider| provider.id() == destination.provider_id)
1705        else {
1706            return Ok(None);
1707        };
1708        // An explicit workspace in the destination config wins over the
1709        // provider default; absence of both means the provider opts out.
1710        let workspace = destination
1711            .config
1712            .get("working_dir")
1713            .and_then(serde_json::Value::as_str)
1714            .map(str::to_string)
1715            .or_else(|| provider.default_workspace());
1716        let Some(workspace) = workspace else {
1717            return Ok(None);
1718        };
1719        let selection = ThreadRunnerSelection {
1720            provider_id: destination.provider_id.clone(),
1721            config: destination.config.clone(),
1722            workspace,
1723            read_roots: Vec::new(),
1724        };
1725        Ok(Some(
1726            self.resolve_thread_runner_binding(thread_id, selection)
1727                .await?,
1728        ))
1729    }
1730
1731    /// Validates a runner selection without creating a thread; the placeholder destination id only appears in error messages.
1732    pub async fn validate_thread_runner_selection(
1733        &self,
1734        selection: ThreadRunnerSelection,
1735    ) -> anyhow::Result<()> {
1736        self.resolve_thread_runner_binding("validate", selection)
1737            .await
1738            .map(|_| ())
1739    }
1740
1741    pub async fn create_thread_with(
1742        &self,
1743        req: CreateThreadRequest,
1744    ) -> anyhow::Result<ThreadMetadata> {
1745        tool_advertisement::validate_external_tool_names(&req.external_tools)?;
1746        let cfg = self.config.read().await.clone();
1747        let now = OffsetDateTime::now_utc();
1748        let workspace = validate_thread_workspace(&req.workspace)?;
1749        let provider = req.provider.unwrap_or(cfg.default_provider);
1750        let model = req.model.unwrap_or(cfg.default_model);
1751        let selection_mode = req
1752            .selection_mode
1753            .unwrap_or_else(|| ModelSelectionMode::manual(provider.clone(), model.clone(), None));
1754        let thread_id = uuid::Uuid::new_v4().to_string();
1755        let runner_binding = match req.runner {
1756            Some(selection) => Some(
1757                self.resolve_thread_runner_binding(&thread_id, selection)
1758                    .await?,
1759            ),
1760            // No explicit per-thread selection: if a runtime-level destination
1761            // is active and its provider advertises a default workspace, bind
1762            // the new thread so its coding tools route into the runner. This is
1763            // what makes a TUI/config-selected runner (e.g. Blaxel) actually
1764            // execute tools in the sandbox instead of locally. Providers that
1765            // return no default workspace stay unbound (legacy local-tools).
1766            None => {
1767                self.synthesize_runtime_runner_binding(
1768                    &thread_id,
1769                    cfg.remote_runner_destination.clone(),
1770                )
1771                .await?
1772            }
1773        };
1774        let runner_destination = runner_binding
1775            .as_ref()
1776            .map(|binding| binding.destination.clone())
1777            .or_else(|| cfg.remote_runner_destination.clone());
1778        let metadata = ThreadMetadata {
1779            thread_id,
1780            title: req.title,
1781            workspace,
1782            workspace_id: req.workspace_id,
1783            root_id: req.root_id,
1784            provider: Some(provider),
1785            model: Some(model),
1786            selection_mode: Some(selection_mode),
1787            tool_allowlist: req.tool_allowlist,
1788            service_tier: req.service_tier,
1789            developer_instructions: req.developer_instructions,
1790            external_tools: req.external_tools,
1791            runner_destination,
1792            runner_state: None,
1793            runner_binding,
1794            created_at: now,
1795            updated_at: now,
1796            message_count: 0,
1797            usage: None,
1798            parent_thread_id: None,
1799            forked_from_turn_id: None,
1800            workspace_fork: None,
1801        };
1802
1803        let metadata = if let Some(store) = &self.thread_store {
1804            store.create_thread(metadata).await?
1805        } else {
1806            metadata
1807        };
1808        self.emit(RoderEvent::ThreadCreated(ThreadCreated {
1809            thread_id: metadata.thread_id.clone(),
1810            timestamp: OffsetDateTime::now_utc(),
1811        }))
1812        .await;
1813        Ok(metadata)
1814    }
1815
1816    pub async fn list_threads(&self) -> anyhow::Result<Vec<ThreadMetadata>> {
1817        if let Some(store) = &self.thread_store {
1818            return store.list_threads().await;
1819        }
1820        Ok(Vec::new())
1821    }
1822
1823    pub async fn list_threads_page(
1824        &self,
1825        options: roder_api::thread::ThreadListOptions,
1826    ) -> anyhow::Result<roder_api::thread::ThreadListPage> {
1827        if let Some(store) = &self.thread_store {
1828            return store.list_threads_page(options).await;
1829        }
1830        Ok(roder_api::thread::ThreadListPage::default())
1831    }
1832
1833    pub async fn load_thread_metadata(
1834        &self,
1835        thread_id: &str,
1836    ) -> anyhow::Result<Option<roder_api::thread::ThreadMetadata>> {
1837        if let Some(store) = &self.thread_store {
1838            return store.load_thread_metadata(&thread_id.to_string()).await;
1839        }
1840        Ok(None)
1841    }
1842
1843    pub async fn archive_thread(&self, thread_id: &str) -> anyhow::Result<bool> {
1844        let archived = if let Some(store) = &self.thread_store {
1845            store.archive_thread(&thread_id.to_string()).await?
1846        } else {
1847            false
1848        };
1849        if archived {
1850            self.thread_events.remove_thread(thread_id).await;
1851            let workspace = self.config.read().await.workspace.clone();
1852            crate::hooks::run_lifecycle(
1853                self,
1854                &thread_id.to_string(),
1855                &SESSION_HOOK_TURN_ID.to_string(),
1856                workspace.as_deref(),
1857                "SessionEnd",
1858                Some("clear"),
1859                serde_json::json!({"reason": "archive"}),
1860            )
1861            .await;
1862            self.thread_item_cache
1863                .lock()
1864                .await
1865                .remove_thread(&thread_id.to_string());
1866            self.evict_runner_session(&thread_id.to_string()).await;
1867        }
1868        Ok(archived)
1869    }
1870
1871    pub async fn start_team(&self, req: TeamStartRequest) -> anyhow::Result<TeamState> {
1872        let cfg = self.config.read().await.clone();
1873        let workspace = self.workspace.display().to_string();
1874        let lead_thread_id = match req.lead_thread_id {
1875            Some(thread_id) => thread_id,
1876            None => {
1877                self.create_thread_with(CreateThreadRequest {
1878                    title: Some("Team lead".to_string()),
1879                    workspace: workspace.clone(),
1880                    workspace_id: None,
1881                    root_id: None,
1882                    provider: None,
1883                    model: None,
1884                    selection_mode: None,
1885                    tool_allowlist: Vec::new(),
1886                    service_tier: None,
1887                    developer_instructions: None,
1888                    external_tools: Vec::new(),
1889                    runner: None,
1890                })
1891                .await?
1892                .thread_id
1893            }
1894        };
1895        let team_id = uuid::Uuid::new_v4().to_string();
1896        let active_lead_turn_id = self.active_turn_for_thread(&lead_thread_id).await;
1897        let lead_selection = match active_lead_turn_id.as_ref() {
1898            Some(turn_id) => self
1899                .active_turn_selections
1900                .read()
1901                .await
1902                .get(turn_id)
1903                .cloned(),
1904            None => self.selection_mode_for_thread(&lead_thread_id).await?,
1905        };
1906        let lead_concrete_selection = lead_selection
1907            .as_ref()
1908            .map(ModelSelectionMode::concrete_selection);
1909        let mut lead = crate::teams::lead_member(
1910            lead_thread_id.clone(),
1911            lead_concrete_selection
1912                .as_ref()
1913                .map(|selection| selection.provider.clone())
1914                .or_else(|| Some(cfg.default_provider.clone())),
1915            lead_concrete_selection
1916                .as_ref()
1917                .map(|selection| selection.model.clone())
1918                .or_else(|| Some(cfg.default_model.clone())),
1919            cfg.policy_mode,
1920        );
1921        if let Some(turn_id) = active_lead_turn_id {
1922            lead.current_turn_id = Some(turn_id);
1923            lead.status = TeamMemberStatus::Running;
1924        }
1925        let mut members = vec![lead];
1926
1927        for (index, member) in req.members.into_iter().enumerate() {
1928            let thread = self
1929                .create_thread_with(CreateThreadRequest {
1930                    title: Some(member.name.clone()),
1931                    workspace: workspace.clone(),
1932                    workspace_id: None,
1933                    root_id: None,
1934                    provider: member.model_provider.clone(),
1935                    model: member.model.clone(),
1936                    selection_mode: None,
1937                    tool_allowlist: Vec::new(),
1938                    service_tier: None,
1939                    developer_instructions: None,
1940                    external_tools: Vec::new(),
1941                    runner: None,
1942                })
1943                .await?;
1944            let member_id = format!("member-{}", index + 1);
1945            let descriptor = crate::teams::teammate_member(
1946                member_id.clone(),
1947                member.name,
1948                thread.thread_id.clone(),
1949                member.model_provider.or(thread.provider),
1950                member.model.or(thread.model),
1951                cfg.policy_mode,
1952            );
1953            members.push(descriptor);
1954        }
1955
1956        let now = OffsetDateTime::now_utc();
1957        let team = self
1958            .teams
1959            .insert(TeamState {
1960                id: team_id.clone(),
1961                lead_thread_id: lead_thread_id.clone(),
1962                display_mode: req.display_mode,
1963                members,
1964                mailbox: Vec::new(),
1965                tasks: Vec::new(),
1966                created_at: now,
1967                updated_at: now,
1968            })
1969            .await?;
1970        self.emit(RoderEvent::TeamStarted(TeamStarted {
1971            team_id: team_id.clone(),
1972            lead_thread_id,
1973            display_mode: team.display_mode,
1974            members: team.members.clone(),
1975            tasks: team.tasks.clone(),
1976            timestamp: OffsetDateTime::now_utc(),
1977        }))
1978        .await;
1979        for member in team
1980            .members
1981            .iter()
1982            .filter(|member| member.role != roder_api::teams::TeamMemberRole::Lead)
1983        {
1984            self.emit(RoderEvent::TeamMemberStarted(TeamMemberStarted {
1985                team_id: team_id.clone(),
1986                member: member.clone(),
1987                timestamp: OffsetDateTime::now_utc(),
1988            }))
1989            .await;
1990        }
1991        Ok(team)
1992    }
1993
1994    pub async fn list_teams(&self) -> Vec<TeamState> {
1995        let active_thread_ids = self
1996            .active_turns
1997            .read()
1998            .await
1999            .values()
2000            .map(|handle| handle.thread_id.clone())
2001            .collect::<std::collections::HashSet<_>>();
2002        self.teams
2003            .list()
2004            .await
2005            .into_iter()
2006            .map(|team| runtime_local_team_view(team, &active_thread_ids))
2007            .collect()
2008    }
2009
2010    pub async fn read_team(&self, team_id: &str) -> Option<TeamState> {
2011        let active_thread_ids = self
2012            .active_turns
2013            .read()
2014            .await
2015            .values()
2016            .map(|handle| handle.thread_id.clone())
2017            .collect::<std::collections::HashSet<_>>();
2018        self.teams
2019            .get(team_id)
2020            .await
2021            .map(|team| runtime_local_team_view(team, &active_thread_ids))
2022    }
2023
2024    pub async fn start_team_member(
2025        &self,
2026        team_id: &str,
2027        req: TeamMemberStartRequest,
2028    ) -> anyhow::Result<TeamState> {
2029        self.start_team_member_with_selection(team_id, req, None)
2030            .await
2031    }
2032
2033    pub async fn message_team_member(
2034        self: &Arc<Self>,
2035        team_id: &str,
2036        member_id: &str,
2037        message: String,
2038    ) -> anyhow::Result<TurnId> {
2039        let team = self
2040            .read_team(team_id)
2041            .await
2042            .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2043        self.followup_team_member(&team.lead_thread_id, team_id, member_id, message)
2044            .await
2045    }
2046
2047    /// Queue a mailbox message without starting an idle agent. If the target is
2048    /// already running, the message is delivered at the next inference boundary.
2049    pub(crate) async fn queue_team_member_message(
2050        self: &Arc<Self>,
2051        caller_thread_id: &ThreadId,
2052        team_id: &str,
2053        member_id: &str,
2054        message: String,
2055    ) -> anyhow::Result<Option<TurnId>> {
2056        let _delivery_guard = self.agent_team_spawn_lock.lock().await;
2057        let team = self
2058            .read_team(team_id)
2059            .await
2060            .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2061        let member = team
2062            .members
2063            .iter()
2064            .find(|member| member.id == member_id)
2065            .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2066            .clone();
2067        if member.status == TeamMemberStatus::Closed {
2068            anyhow::bail!("subagent {} is closed", member.name);
2069        }
2070        let from_member_id = team
2071            .members
2072            .iter()
2073            .find(|candidate| candidate.thread_id == *caller_thread_id)
2074            .map(|candidate| candidate.id.clone());
2075        self.teams
2076            .append_mailbox_message(
2077                team_id,
2078                from_member_id,
2079                member_id.to_string(),
2080                TeamMailboxMessageKind::Message,
2081                message,
2082            )
2083            .await?;
2084        let Some(turn_id) = self.active_turn_for_thread(&member.thread_id).await else {
2085            return Ok(None);
2086        };
2087        self.deliver_pending_team_mailbox(team_id, member_id, member.thread_id, turn_id.clone())
2088            .await?;
2089        Ok(Some(turn_id))
2090    }
2091
2092    /// Deliver queued messages and start an idle agent, or steer its active turn.
2093    pub(crate) async fn followup_team_member(
2094        self: &Arc<Self>,
2095        caller_thread_id: &ThreadId,
2096        team_id: &str,
2097        member_id: &str,
2098        message: String,
2099    ) -> anyhow::Result<TurnId> {
2100        let _spawn_guard = self.agent_team_spawn_lock.lock().await;
2101        self.followup_team_member_locked(caller_thread_id, team_id, member_id, message)
2102            .await
2103    }
2104
2105    async fn followup_team_member_locked(
2106        self: &Arc<Self>,
2107        caller_thread_id: &ThreadId,
2108        team_id: &str,
2109        member_id: &str,
2110        message: String,
2111    ) -> anyhow::Result<TurnId> {
2112        let team = self
2113            .read_team(team_id)
2114            .await
2115            .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2116        let member = team
2117            .members
2118            .iter()
2119            .find(|member| member.id == member_id)
2120            .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2121            .clone();
2122        if member.status == TeamMemberStatus::Closed {
2123            anyhow::bail!("subagent {} is closed", member.name);
2124        }
2125        let from_member_id = team
2126            .members
2127            .iter()
2128            .find(|candidate| candidate.thread_id == *caller_thread_id)
2129            .map(|candidate| candidate.id.clone());
2130        self.teams
2131            .append_mailbox_message(
2132                team_id,
2133                from_member_id,
2134                member_id.to_string(),
2135                TeamMailboxMessageKind::NewTask,
2136                message,
2137            )
2138            .await?;
2139        if let Some(turn_id) = self.active_turn_for_thread(&member.thread_id).await {
2140            self.deliver_pending_team_mailbox(
2141                team_id,
2142                member_id,
2143                member.thread_id,
2144                turn_id.clone(),
2145            )
2146            .await?;
2147            return Ok(turn_id);
2148        }
2149
2150        self.ensure_codex_v2_team_capacity(&team, &member.id, None)
2151            .await?;
2152        let metadata = self.load_thread_metadata(&member.thread_id).await?;
2153        let inherited_context = self
2154            .team_member_turn_contexts
2155            .lock()
2156            .await
2157            .get(&member.thread_id)
2158            .cloned();
2159        let workspace = inherited_context
2160            .as_ref()
2161            .map(|context| context.workspace.clone())
2162            .or_else(|| metadata.as_ref().map(|metadata| metadata.workspace.clone()))
2163            .unwrap_or_else(|| self.workspace.display().to_string());
2164        let member_thread_id = member.thread_id;
2165        let turn_id = self
2166            .start_turn(StartTurnRequest {
2167                thread_id: member_thread_id.clone(),
2168                message: String::new(),
2169                images: Vec::new(),
2170                provider_override: member.model_provider,
2171                model_override: member.model,
2172                reasoning_override: metadata
2173                    .as_ref()
2174                    .and_then(|metadata| metadata.selection_mode.as_ref())
2175                    .and_then(ModelSelectionMode::reasoning)
2176                    .map(str::to_string),
2177                workspace,
2178                instructions: inherited_context
2179                    .as_ref()
2180                    .map(|context| context.instructions.clone())
2181                    .unwrap_or_else(crate::default_instructions),
2182                developer_context: inherited_context
2183                    .as_ref()
2184                    .and_then(|context| context.developer_context.clone()),
2185                task_ledger_required: false,
2186                service_tier_override: None,
2187            })
2188            .await?;
2189        Ok(turn_id)
2190    }
2191
2192    pub async fn set_team_member_policy_mode(
2193        &self,
2194        team_id: &str,
2195        member_id: &str,
2196        policy_mode: PolicyMode,
2197    ) -> anyhow::Result<TeamState> {
2198        self.teams
2199            .set_member_policy_mode(team_id, member_id, policy_mode)
2200            .await
2201    }
2202
2203    pub async fn interrupt_team_member(
2204        &self,
2205        team_id: &str,
2206        member_id: &str,
2207    ) -> anyhow::Result<Option<TurnId>> {
2208        let _interrupt_guard = self.agent_team_spawn_lock.lock().await;
2209        let team = self
2210            .read_team(team_id)
2211            .await
2212            .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2213        let member = team
2214            .members
2215            .iter()
2216            .find(|member| member.id == member_id)
2217            .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2218            .clone();
2219        if member.status != TeamMemberStatus::Running {
2220            return Ok(None);
2221        }
2222        let Some(turn_id) = member.current_turn_id.clone() else {
2223            return Ok(None);
2224        };
2225        if self
2226            .active_turn_for_thread(&member.thread_id)
2227            .await
2228            .as_ref()
2229            != Some(&turn_id)
2230        {
2231            return Ok(None);
2232        }
2233        self.interrupt_turn(member.thread_id.clone(), turn_id.clone())
2234            .await?;
2235        // Make queued mailbox work immediately eligible for the replacement turn while the
2236        // lifecycle lock still prevents a follow-up from starting. Delivery acknowledgements
2237        // are turn-owned, so a late acknowledgement from the aborted turn cannot steal a
2238        // reservation acquired by its replacement.
2239        self.teams
2240            .release_mailbox_reservations_for_turn(&turn_id)
2241            .await;
2242        self.teams
2243            .update_member(team_id, member_id, |member| {
2244                member.status = TeamMemberStatus::Interrupted;
2245                member.current_turn_id = None;
2246                member.terminal_error = None;
2247            })
2248            .await?;
2249        self.emit(RoderEvent::TeamMemberCompleted(TeamMemberCompleted {
2250            team_id: team_id.to_string(),
2251            member_id: member_id.to_string(),
2252            member_thread_id: member.thread_id,
2253            turn_id: Some(turn_id.clone()),
2254            status: TeamMemberStatus::Interrupted,
2255            final_message: member.final_message,
2256            error: None,
2257            timestamp: OffsetDateTime::now_utc(),
2258        }))
2259        .await;
2260        Ok(Some(turn_id))
2261    }
2262
2263    pub async fn close_team_member(
2264        &self,
2265        team_id: &str,
2266        member_id: &str,
2267    ) -> anyhow::Result<roder_api::teams::TeamMemberDescriptor> {
2268        let team = self
2269            .read_team(team_id)
2270            .await
2271            .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2272        let member = team
2273            .members
2274            .iter()
2275            .find(|member| member.id == member_id)
2276            .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2277            .clone();
2278        if member.role == roder_api::teams::TeamMemberRole::Lead {
2279            anyhow::bail!("team lead cannot be closed as a subagent");
2280        }
2281        let interrupted_turn_id = if member.status == TeamMemberStatus::Running {
2282            if let Some(turn_id) = member.current_turn_id.clone() {
2283                self.interrupt_turn(member.thread_id.clone(), turn_id.clone())
2284                    .await?;
2285                Some(turn_id)
2286            } else {
2287                None
2288            }
2289        } else {
2290            member.current_turn_id.clone()
2291        };
2292        let updated = self
2293            .teams
2294            .update_member(team_id, member_id, |member| {
2295                member.status = TeamMemberStatus::Closed;
2296                member.current_turn_id = None;
2297            })
2298            .await?;
2299        let closed = updated
2300            .members
2301            .iter()
2302            .find(|member| member.id == member_id)
2303            .cloned()
2304            .ok_or_else(|| anyhow::anyhow!("closed team member disappeared"))?;
2305        self.team_member_turn_contexts
2306            .lock()
2307            .await
2308            .remove(&closed.thread_id);
2309        self.thread_events.remove_thread(&closed.thread_id).await;
2310        self.emit(RoderEvent::TeamMemberCompleted(TeamMemberCompleted {
2311            team_id: team_id.to_string(),
2312            member_id: closed.id.clone(),
2313            member_thread_id: closed.thread_id.clone(),
2314            turn_id: interrupted_turn_id,
2315            status: TeamMemberStatus::Closed,
2316            final_message: closed.final_message.clone(),
2317            error: closed.terminal_error.clone(),
2318            timestamp: OffsetDateTime::now_utc(),
2319        }))
2320        .await;
2321        Ok(closed)
2322    }
2323
2324    pub async fn cleanup_team(&self, team_id: &str, force: bool) -> anyhow::Result<bool> {
2325        let Some(team) = self.read_team(team_id).await else {
2326            return Ok(false);
2327        };
2328        if !force
2329            && team
2330                .members
2331                .iter()
2332                .any(|member| member.status == TeamMemberStatus::Running)
2333        {
2334            anyhow::bail!("team {team_id:?} has active teammates; use forced cleanup");
2335        }
2336        if force {
2337            for member in team.members.iter().filter(|member| {
2338                member.role != roder_api::teams::TeamMemberRole::Lead
2339                    && member.status == TeamMemberStatus::Running
2340            }) {
2341                if let Some(turn_id) = member
2342                    .current_turn_id
2343                    .clone()
2344                    .or(self.active_turn_for_thread(&member.thread_id).await)
2345                {
2346                    let _ = self.interrupt_turn(member.thread_id.clone(), turn_id).await;
2347                }
2348            }
2349        }
2350        let removed = self.teams.remove(team_id).await?.is_some();
2351        if removed {
2352            let member_thread_ids = team
2353                .members
2354                .iter()
2355                .map(|member| member.thread_id.clone())
2356                .collect::<std::collections::HashSet<_>>();
2357            self.team_member_turn_contexts
2358                .lock()
2359                .await
2360                .retain(|thread_id, _| !member_thread_ids.contains(thread_id));
2361            self.emit(RoderEvent::TeamCleanupCompleted(TeamCleanupCompleted {
2362                team_id: team_id.to_string(),
2363                forced: force,
2364                timestamp: OffsetDateTime::now_utc(),
2365            }))
2366            .await;
2367        }
2368        Ok(removed)
2369    }
2370
2371    pub async fn effective_policy_mode_for_thread(&self, thread_id: &str) -> PolicyMode {
2372        if let Some(mode) = self.teams.policy_mode_for_thread(thread_id).await {
2373            return mode;
2374        }
2375        match self.thread_policy_mode(thread_id).await {
2376            Ok(Some(mode)) => mode,
2377            Ok(None) => self.status().await.policy_mode,
2378            Err(error) => {
2379                eprintln!("could not read thread policy for {thread_id}: {error}; using plan mode");
2380                PolicyMode::Plan
2381            }
2382        }
2383    }
2384
2385    async fn complete_team_member_turn_with_result(
2386        &self,
2387        thread_id: &ThreadId,
2388        turn_id: &TurnId,
2389        status: TeamMemberStatus,
2390        final_message: Option<String>,
2391        terminal_error: Option<String>,
2392    ) -> anyhow::Result<()> {
2393        let _delivery_guard = self.agent_team_spawn_lock.lock().await;
2394        let Some((team_id, member)) = self
2395            .teams
2396            .complete_member_turn(
2397                thread_id,
2398                turn_id,
2399                status,
2400                final_message.clone(),
2401                terminal_error.clone(),
2402            )
2403            .await?
2404        else {
2405            return Ok(());
2406        };
2407        if let Some(parent_thread_id) = member.parent_thread_id.as_ref()
2408            && let Some(team) = self.read_team(&team_id).await
2409            && let Some(parent) = team
2410                .members
2411                .iter()
2412                .find(|candidate| candidate.thread_id == *parent_thread_id)
2413        {
2414            let identity = member
2415                .agent_path
2416                .as_deref()
2417                .or(member.task_name.as_deref())
2418                .unwrap_or(&member.name);
2419            let mut report = format!("Agent {identity} finished with status {status:?}.");
2420            if let Some(message) = final_message.as_deref()
2421                && !message.trim().is_empty()
2422            {
2423                report.push_str("\n\nFinal result:\n");
2424                report.push_str(message);
2425            }
2426            if let Some(error) = terminal_error.as_deref()
2427                && !error.trim().is_empty()
2428            {
2429                report.push_str("\n\nTerminal error:\n");
2430                report.push_str(error);
2431            }
2432            let mailbox_appended = self
2433                .teams
2434                .append_mailbox_message(
2435                    &team_id,
2436                    Some(member.id.clone()),
2437                    parent.id.clone(),
2438                    TeamMailboxMessageKind::FinalAnswer,
2439                    report,
2440                )
2441                .await
2442                .is_ok();
2443            if mailbox_appended
2444                && let Some(parent_turn_id) = self.active_turn_for_thread(parent_thread_id).await
2445            {
2446                // The parent may finish between the active-turn lookup and enqueue. Its durable
2447                // mailbox entry remains pending for the next turn, while terminal observers must
2448                // still receive TeamMemberCompleted for this child.
2449                let _ = self
2450                    .deliver_pending_team_mailbox(
2451                        &team_id,
2452                        &parent.id,
2453                        parent_thread_id.clone(),
2454                        parent_turn_id,
2455                    )
2456                    .await;
2457            }
2458        }
2459        self.emit(RoderEvent::TeamMemberCompleted(TeamMemberCompleted {
2460            team_id,
2461            member_id: member.id,
2462            member_thread_id: member.thread_id,
2463            turn_id: Some(turn_id.clone()),
2464            status,
2465            final_message,
2466            error: terminal_error,
2467            timestamp: OffsetDateTime::now_utc(),
2468        }))
2469        .await;
2470        Ok(())
2471    }
2472
2473    /// Stops accepting new turns, requests interruption for every active turn,
2474    /// and waits for their runtime tasks to reach terminal cleanup up to
2475    /// `timeout`. Repeated calls are safe and continue draining the same set.
2476    pub async fn drain_active_turns(&self, timeout: std::time::Duration) -> RuntimeDrainOutcome {
2477        let started_at = tokio::time::Instant::now();
2478        let persistence_failures_before =
2479            self.lifecycle_persistence_failures.load(Ordering::Acquire);
2480        let turn_admission = self.turn_admission.lock().await;
2481        self.accepting_turns.store(false, Ordering::Release);
2482        let active = self
2483            .active_turns
2484            .read()
2485            .await
2486            .iter()
2487            .map(|(turn_id, handle)| (turn_id.clone(), handle.thread_id.clone()))
2488            .collect::<Vec<_>>();
2489        let draining = self
2490            .turn_drains
2491            .read()
2492            .await
2493            .iter()
2494            .map(|(turn_id, drain)| (turn_id.clone(), drain.thread_id.clone()))
2495            .collect::<Vec<_>>();
2496        drop(turn_admission);
2497
2498        let mut interrupted_turns = std::collections::BTreeMap::new();
2499        for (turn_id, thread_id) in active.iter().chain(draining.iter()) {
2500            interrupted_turns.insert(turn_id.clone(), thread_id.clone());
2501        }
2502        let interrupted_turn_ids = interrupted_turns.keys().cloned().collect::<Vec<_>>();
2503        for (turn_id, thread_id) in active {
2504            let _ = self
2505                .interrupt_turn_with_reason(thread_id, turn_id, TurnLifecycleReason::Shutdown)
2506                .await;
2507        }
2508
2509        let wait_for_empty = async {
2510            loop {
2511                let notified = self.active_turns_changed.notified();
2512                if self.active_turns.read().await.is_empty()
2513                    && self.turn_drains.read().await.is_empty()
2514                {
2515                    return;
2516                }
2517                notified.await;
2518            }
2519        };
2520        let outcome = if tokio::time::timeout(timeout, wait_for_empty).await.is_ok() {
2521            if self.lifecycle_persistence_failures.load(Ordering::Acquire)
2522                > persistence_failures_before
2523            {
2524                RuntimeDrainOutcome::PersistenceFailed {
2525                    interrupted_turn_ids,
2526                    remaining_turn_ids: Vec::new(),
2527                }
2528            } else {
2529                RuntimeDrainOutcome::Clean {
2530                    interrupted_turn_ids,
2531                }
2532            }
2533        } else {
2534            let active_remaining = self
2535                .active_turns
2536                .read()
2537                .await
2538                .iter()
2539                .map(|(turn_id, handle)| (turn_id.clone(), handle.drain.clone()))
2540                .collect::<Vec<_>>();
2541            let draining_remaining = self
2542                .turn_drains
2543                .read()
2544                .await
2545                .iter()
2546                .map(|(turn_id, drain)| (turn_id.clone(), drain.clone()))
2547                .collect::<Vec<_>>();
2548            let remaining = active_remaining
2549                .into_iter()
2550                .chain(draining_remaining)
2551                .collect::<std::collections::BTreeMap<_, _>>();
2552            let remaining_turn_ids = remaining.keys().cloned().collect::<Vec<_>>();
2553            for turn_id in &remaining_turn_ids {
2554                let handle = remaining
2555                    .get(turn_id)
2556                    .expect("remaining turn ID must have a drain handle");
2557                self.record_turn_lifecycle(
2558                    handle.thread_id.clone(),
2559                    turn_id.clone(),
2560                    TurnLifecycleState::InterruptRequested,
2561                    TurnCleanupState::TimedOut,
2562                    Some(TurnLifecycleReason::Shutdown),
2563                )
2564                .await;
2565            }
2566            if self.lifecycle_persistence_failures.load(Ordering::Acquire)
2567                > persistence_failures_before
2568            {
2569                RuntimeDrainOutcome::PersistenceFailed {
2570                    interrupted_turn_ids,
2571                    remaining_turn_ids,
2572                }
2573            } else {
2574                RuntimeDrainOutcome::DeadlineExceeded {
2575                    interrupted_turn_ids,
2576                    remaining_turn_ids,
2577                }
2578            }
2579        };
2580        self.record_lifecycle_drain_outcome(started_at, &outcome);
2581        outcome
2582    }
2583
2584    /// Enables turn submission after a prior drain attempt. This is intended
2585    /// for embedders that keep a runtime alive after a bounded drain timeout.
2586    pub fn resume_accepting_turns(&self) {
2587        self.accepting_turns.store(true, Ordering::Release);
2588    }
2589
2590    pub async fn turn_lifecycle_snapshot(
2591        &self,
2592        thread_id: &ThreadId,
2593    ) -> anyhow::Result<TurnLifecycleSnapshot> {
2594        let Some(store) = &self.thread_store else {
2595            return Ok(TurnLifecycleSnapshot::default());
2596        };
2597        let extension_states = store.load_extension_states(thread_id).await?;
2598        Ok(turn_lifecycle_snapshot(&extension_states))
2599    }
2600
2601    pub async fn load_thread(
2602        &self,
2603        thread_id: &ThreadId,
2604    ) -> anyhow::Result<Option<ThreadSnapshot>> {
2605        let loaded = if let Some(store) = &self.thread_store {
2606            store.load_thread(thread_id).await?
2607        } else {
2608            None
2609        };
2610        if let Some(snapshot) = loaded.as_ref() {
2611            let live_turn_ids = self
2612                .active_turns
2613                .read()
2614                .await
2615                .keys()
2616                .cloned()
2617                .chain(self.turn_drains.read().await.keys().cloned())
2618                .collect::<std::collections::HashSet<_>>();
2619            let lifecycle = turn_lifecycle_snapshot(&snapshot.extension_states);
2620            for record in lifecycle.records.into_iter().filter(|record| {
2621                record.state.requires_recovery() && !live_turn_ids.contains(&record.turn_id)
2622            }) {
2623                self.lifecycle_restart_reconciliations
2624                    .fetch_add(1, Ordering::AcqRel);
2625                self.record_turn_lifecycle(
2626                    record.thread_id,
2627                    record.turn_id,
2628                    TurnLifecycleState::RecoveryNeeded,
2629                    TurnCleanupState::Unknown,
2630                    Some(TurnLifecycleReason::RuntimeRestart),
2631                )
2632                .await;
2633            }
2634            self.emit(RoderEvent::ThreadLoaded(ThreadLoaded {
2635                thread_id: thread_id.clone(),
2636                timestamp: OffsetDateTime::now_utc(),
2637            }))
2638            .await;
2639        }
2640        Ok(loaded)
2641    }
2642
2643    pub async fn workspace_for_thread(&self, thread_id: &ThreadId) -> anyhow::Result<String> {
2644        if let Some(store) = &self.thread_store {
2645            let snapshot = store
2646                .load_thread(thread_id)
2647                .await?
2648                .ok_or_else(|| anyhow::anyhow!("thread not found: {thread_id}"));
2649            match snapshot {
2650                Ok(snapshot) => {
2651                    if let Some(metadata) = snapshot.metadata {
2652                        // Fork-backed threads fail closed before any write
2653                        // when their workspace was removed out-of-band.
2654                        if let Some(fork) = &metadata.workspace_fork
2655                            && fork.status == roder_api::forks::ForkStatus::Active
2656                            && !std::path::Path::new(&metadata.workspace).is_dir()
2657                        {
2658                            anyhow::bail!(
2659                                "workspace fork {} is missing its workspace at {}; restore it or \
2660                                 remove the fork before running turns in this thread",
2661                                fork.id,
2662                                metadata.workspace
2663                            );
2664                        }
2665                        return Ok(metadata.workspace);
2666                    }
2667                    eprintln!(
2668                        "thread metadata missing while resolving workspace for {thread_id}; falling back to runtime workspace"
2669                    );
2670                }
2671                Err(err) => {
2672                    eprintln!(
2673                        "thread missing while resolving workspace for {thread_id}: {err}; falling back to runtime workspace"
2674                    );
2675                }
2676            }
2677        }
2678        Ok(self.workspace.display().to_string())
2679    }
2680
2681    pub(crate) async fn selection_mode_for_thread(
2682        &self,
2683        thread_id: &ThreadId,
2684    ) -> anyhow::Result<Option<ModelSelectionMode>> {
2685        let Some(store) = &self.thread_store else {
2686            return Ok(None);
2687        };
2688        Ok(store
2689            .load_thread_metadata(thread_id)
2690            .await?
2691            .and_then(|metadata| {
2692                metadata
2693                    .selection_mode
2694                    .or_else(|| match (metadata.provider, metadata.model) {
2695                        (Some(provider), Some(model)) => {
2696                            Some(ModelSelectionMode::manual(provider, model, None))
2697                        }
2698                        _ => None,
2699                    })
2700            }))
2701    }
2702
2703    /// Concrete provider/model for configured-role subagents (`task`,
2704    /// `agent_swarm`). Prefers the live parent turn selection so children stay
2705    /// on SuperGrok/Grok (or whatever the lead is using) instead of process
2706    /// startup defaults such as openai/gpt-5.5.
2707    pub(crate) async fn parent_model_selection_for_subagents(
2708        &self,
2709        thread_id: &ThreadId,
2710        turn_id: &TurnId,
2711    ) -> Option<roder_api::inference::ModelSelection> {
2712        if let Some(selection) = self
2713            .active_turn_selections
2714            .read()
2715            .await
2716            .get(turn_id)
2717            .cloned()
2718        {
2719            return Some(selection.concrete_selection());
2720        }
2721        if let Ok(Some(selection)) = self.selection_mode_for_thread(thread_id).await {
2722            return Some(selection.concrete_selection());
2723        }
2724        let cfg = self.status().await;
2725        if cfg.default_provider.trim().is_empty() || cfg.default_model.trim().is_empty() {
2726            return None;
2727        }
2728        Some(roder_api::inference::ModelSelection {
2729            provider: cfg.default_provider,
2730            model: cfg.default_model,
2731        })
2732    }
2733
2734    /// Per-thread tool allowlist, developer instructions, and external tools persisted at thread creation.
2735    pub(crate) async fn thread_turn_overrides(
2736        &self,
2737        thread_id: &ThreadId,
2738    ) -> anyhow::Result<ThreadTurnOverrides> {
2739        let Some(store) = &self.thread_store else {
2740            return Ok(ThreadTurnOverrides::default());
2741        };
2742        Ok(store
2743            .load_thread_metadata(thread_id)
2744            .await?
2745            .map(|metadata| ThreadTurnOverrides {
2746                service_tier: metadata.service_tier,
2747                tool_allowlist: metadata.tool_allowlist,
2748                developer_instructions: metadata.developer_instructions,
2749                external_tools: metadata.external_tools,
2750            })
2751            .unwrap_or_default())
2752    }
2753
2754    pub async fn set_thread_selection_mode(
2755        &self,
2756        thread_id: &ThreadId,
2757        selection_mode: ModelSelectionMode,
2758    ) -> anyhow::Result<()> {
2759        let Some(store) = &self.thread_store else {
2760            return Ok(());
2761        };
2762        let Some(mut metadata) = store.load_thread_metadata(thread_id).await? else {
2763            anyhow::bail!("thread not found: {thread_id}");
2764        };
2765        let concrete = selection_mode.concrete_selection();
2766        metadata.provider = Some(concrete.provider);
2767        metadata.model = Some(concrete.model);
2768        metadata.selection_mode = Some(selection_mode);
2769        metadata.updated_at = OffsetDateTime::now_utc();
2770        store.update_thread_metadata(metadata).await?;
2771        Ok(())
2772    }
2773
2774    async fn runner_session_for_thread(
2775        &self,
2776        thread_id: &ThreadId,
2777    ) -> anyhow::Result<Option<(RunnerDestination, Arc<dyn RemoteRunnerSession>)>> {
2778        let metadata = if let Some(store) = &self.thread_store {
2779            store.load_thread_metadata(thread_id).await?
2780        } else {
2781            None
2782        };
2783        // An explicit per-thread binding wins over the runtime-level destination.
2784        let destination = metadata
2785            .as_ref()
2786            .and_then(|metadata| metadata.runner_binding.as_ref())
2787            .map(|binding| binding.destination.clone())
2788            .or(self.config.read().await.remote_runner_destination.clone());
2789        let Some(destination) = destination else {
2790            self.evict_runner_session(thread_id).await;
2791            return Ok(None);
2792        };
2793        let provider = self
2794            .registry
2795            .remote_runner_providers
2796            .iter()
2797            .find(|provider| provider.id() == destination.provider_id)
2798            .cloned()
2799            .ok_or_else(|| {
2800                anyhow::anyhow!(
2801                    "remote runner provider {:?} is not installed",
2802                    destination.provider_id
2803                )
2804            })?;
2805        let persisted_state = metadata.and_then(|metadata| metadata.runner_state);
2806        let slot = {
2807            let mut sessions = self.runner_sessions.lock().await;
2808            match sessions.get(thread_id) {
2809                Some(slot) if slot.matches(&destination) => slot.clone(),
2810                _ => {
2811                    let slot = Arc::new(RunnerSessionSlot::empty(&destination));
2812                    sessions.insert(thread_id.clone(), slot.clone());
2813                    slot
2814                }
2815            }
2816        };
2817
2818        loop {
2819            let initialized = slot.initialized.notified();
2820            let mut state = slot.state.lock().await;
2821            match &*state {
2822                RunnerSessionSlotState::Ready(cached) => {
2823                    return Ok(Some((cached.destination.clone(), cached.session.clone())));
2824                }
2825                RunnerSessionSlotState::Failed(error) => {
2826                    return Err(anyhow::anyhow!(error.to_string()));
2827                }
2828                RunnerSessionSlotState::Initializing => {
2829                    drop(state);
2830                    initialized.await;
2831                }
2832                RunnerSessionSlotState::Empty => {
2833                    *state = RunnerSessionSlotState::Initializing;
2834                    drop(state);
2835                    self.spawn_runner_session_initialization(
2836                        thread_id.clone(),
2837                        destination.clone(),
2838                        provider.clone(),
2839                        persisted_state.clone(),
2840                        slot.clone(),
2841                    );
2842                }
2843            }
2844        }
2845    }
2846
2847    fn spawn_runner_session_initialization(
2848        &self,
2849        thread_id: ThreadId,
2850        destination: RunnerDestination,
2851        provider: Arc<dyn RemoteRunnerProvider>,
2852        persisted_state: Option<RunnerSessionState>,
2853        slot: Arc<RunnerSessionSlot>,
2854    ) {
2855        let thread_store = self.thread_store.clone();
2856        let runner_sessions = self.runner_sessions.clone();
2857        // The task intentionally outlives the turn that first requested the
2858        // workspace. Interrupting that turn must not strand the slot in
2859        // `Initializing` and deadlock every later tool or lifecycle call.
2860        drop(tokio::spawn(async move {
2861            let initialized = async {
2862                let session = if let Some(state) = persisted_state
2863                    && state.provider_id == destination.provider_id
2864                    && state.destination_id == destination.id
2865                {
2866                    match provider.resume_session(state).await {
2867                        Ok(session) => session,
2868                        Err(_) => provider.create_session(destination.clone()).await?,
2869                    }
2870                } else {
2871                    provider.create_session(destination.clone()).await?
2872                };
2873                let resolved = (destination.clone(), session.clone());
2874                // Persist before releasing the singleflight or dispatching
2875                // the first tool. A later process can rejoin even if the turn
2876                // that requested initialization has already been interrupted.
2877                Self::persist_runner_state_in_store(
2878                    thread_store.as_ref(),
2879                    &thread_id,
2880                    Some(&resolved),
2881                )
2882                .await?;
2883                Ok::<_, anyhow::Error>(CachedRunnerSession {
2884                    destination,
2885                    session,
2886                })
2887            }
2888            .await;
2889
2890            match initialized {
2891                Ok(cached) => {
2892                    *slot.state.lock().await = RunnerSessionSlotState::Ready(cached);
2893                    slot.initialized.notify_waiters();
2894                }
2895                Err(error) => {
2896                    let message: Arc<str> = Arc::from(format!("{error:#}"));
2897                    *slot.state.lock().await = RunnerSessionSlotState::Failed(message);
2898                    slot.initialized.notify_waiters();
2899                    let mut sessions = runner_sessions.lock().await;
2900                    let is_current = sessions
2901                        .get(&thread_id)
2902                        .is_some_and(|current| Arc::ptr_eq(current, &slot));
2903                    if is_current {
2904                        sessions.remove(&thread_id);
2905                    }
2906                }
2907            }
2908        }));
2909    }
2910
2911    async fn evict_runner_session(&self, thread_id: &ThreadId) {
2912        self.runner_sessions.lock().await.remove(thread_id);
2913    }
2914
2915    async fn cache_runner_session(
2916        &self,
2917        thread_id: &ThreadId,
2918        destination: RunnerDestination,
2919        session: Arc<dyn RemoteRunnerSession>,
2920    ) {
2921        let cached = CachedRunnerSession {
2922            destination,
2923            session,
2924        };
2925        self.runner_sessions.lock().await.insert(
2926            thread_id.clone(),
2927            Arc::new(RunnerSessionSlot::ready(cached)),
2928        );
2929    }
2930
2931    /// Read a thread's explicit runner binding without creating or resuming a
2932    /// session. Tool routing uses this before local previews so runner-backed
2933    /// and fail-closed hosted calls never inspect the host workspace.
2934    pub(crate) async fn runner_binding_for_thread(
2935        &self,
2936        thread_id: &ThreadId,
2937    ) -> anyhow::Result<Option<ThreadRunnerBinding>> {
2938        let Some(store) = &self.thread_store else {
2939            return Ok(None);
2940        };
2941        Ok(store
2942            .load_thread_metadata(thread_id)
2943            .await?
2944            .and_then(|metadata| metadata.runner_binding))
2945    }
2946
2947    /// Resolve a previously-read binding into a live remote workspace. The
2948    /// session remains lazy: callers invoke this only after policy approval.
2949    pub(crate) async fn remote_workspace_for_binding(
2950        &self,
2951        thread_id: &ThreadId,
2952        binding: ThreadRunnerBinding,
2953    ) -> anyhow::Result<Arc<RemoteWorkspace>> {
2954        let session = self
2955            .runner_session_for_thread(thread_id)
2956            .await?
2957            .map(|(_, session)| session)
2958            .ok_or_else(|| {
2959                anyhow::anyhow!("runner-bound thread {thread_id} has no runner session")
2960            })?;
2961        Ok(Arc::new(RemoteWorkspace {
2962            session,
2963            root: binding.workspace,
2964            read_roots: binding.read_roots,
2965        }))
2966    }
2967
2968    pub(crate) async fn runner_tool_execution_lock(
2969        &self,
2970        session: &dyn RemoteRunnerSession,
2971    ) -> Arc<RunnerToolExecutionMutex> {
2972        let state = session.state();
2973        let key = (state.provider_id, state.session_id);
2974        let mut locks = self.runner_tool_execution_locks.lock().await;
2975        if let Some(lock) = locks.get(&key).and_then(Weak::upgrade) {
2976            return lock;
2977        }
2978
2979        locks.retain(|_, lock| lock.strong_count() > 0);
2980        let lock = Arc::new(Mutex::new(()));
2981        locks.insert(key, Arc::downgrade(&lock));
2982        lock
2983    }
2984
2985    async fn persist_runner_state(
2986        &self,
2987        thread_id: &ThreadId,
2988        runner: Option<&(RunnerDestination, Arc<dyn RemoteRunnerSession>)>,
2989    ) -> anyhow::Result<()> {
2990        Self::persist_runner_state_in_store(self.thread_store.as_ref(), thread_id, runner).await
2991    }
2992
2993    async fn persist_runner_state_in_store(
2994        store: Option<&Arc<dyn ThreadStore>>,
2995        thread_id: &ThreadId,
2996        runner: Option<&(RunnerDestination, Arc<dyn RemoteRunnerSession>)>,
2997    ) -> anyhow::Result<()> {
2998        let Some((destination, session)) = runner else {
2999            return Ok(());
3000        };
3001        let Some(store) = store else {
3002            return Ok(());
3003        };
3004        let Some(snapshot) = store.load_thread(thread_id).await? else {
3005            return Ok(());
3006        };
3007        let Some(mut metadata) = snapshot.metadata else {
3008            return Ok(());
3009        };
3010        metadata.runner_destination = Some(destination.clone());
3011        metadata.runner_state = Some(session.state());
3012        metadata.updated_at = OffsetDateTime::now_utc();
3013        store.update_thread_metadata(metadata).await?;
3014        Ok(())
3015    }
3016
3017    fn remote_runner_provider_by_id(
3018        &self,
3019        provider_id: &str,
3020    ) -> Option<Arc<dyn RemoteRunnerProvider>> {
3021        self.registry
3022            .remote_runner_providers
3023            .iter()
3024            .find(|provider| provider.id() == provider_id)
3025            .cloned()
3026    }
3027
3028    /// Resolve the live runner session for a thread along with its provider,
3029    /// failing clearly when the thread is not runner-bound.
3030    async fn thread_runner_session(
3031        &self,
3032        thread_id: &ThreadId,
3033    ) -> anyhow::Result<(
3034        RunnerDestination,
3035        Arc<dyn RemoteRunnerProvider>,
3036        Arc<dyn RemoteRunnerSession>,
3037    )> {
3038        let Some((destination, session)) = self.runner_session_for_thread(thread_id).await? else {
3039            anyhow::bail!("thread {thread_id} is not bound to a remote runner");
3040        };
3041        let provider = self
3042            .remote_runner_provider_by_id(&destination.provider_id)
3043            .ok_or_else(|| {
3044                anyhow::anyhow!(
3045                    "remote runner provider {:?} is not installed",
3046                    destination.provider_id
3047                )
3048            })?;
3049        Ok((destination, provider, session))
3050    }
3051
3052    /// Pause a runner-bound thread's session toward standby and persist the
3053    /// post-pause state. Errors if the provider is not pausable.
3054    pub async fn pause_thread_runner(
3055        &self,
3056        thread_id: &ThreadId,
3057    ) -> anyhow::Result<RunnerSessionState> {
3058        let (destination, provider, session) = self.thread_runner_session(thread_id).await?;
3059        anyhow::ensure!(
3060            provider.capabilities().pausable,
3061            "remote runner provider {:?} does not support pausing",
3062            destination.provider_id
3063        );
3064        let state = session.pause().await?;
3065        self.persist_runner_state(thread_id, Some(&(destination, session)))
3066            .await?;
3067        Ok(state)
3068    }
3069
3070    /// Resume (wake) a runner-bound thread's paused session and persist state.
3071    pub async fn resume_thread_runner(
3072        &self,
3073        thread_id: &ThreadId,
3074    ) -> anyhow::Result<RunnerSessionState> {
3075        let (destination, _provider, session) = self.thread_runner_session(thread_id).await?;
3076        let state = session.resume().await?;
3077        self.persist_runner_state(thread_id, Some(&(destination, session)))
3078            .await?;
3079        Ok(state)
3080    }
3081
3082    /// Detach a runner-bound thread's session: persist the durable, rejoinable
3083    /// state and leave the remote sandbox alive. Errors if not detachable.
3084    pub async fn detach_thread_runner(
3085        &self,
3086        thread_id: &ThreadId,
3087    ) -> anyhow::Result<RunnerSessionState> {
3088        let (destination, provider, session) = self.thread_runner_session(thread_id).await?;
3089        anyhow::ensure!(
3090            provider.capabilities().detachable,
3091            "remote runner provider {:?} does not support detaching",
3092            destination.provider_id
3093        );
3094        let state = session.detach().await?;
3095        // Persist the detached state explicitly so a later turn or process
3096        // rejoins the same sandbox instead of provisioning a new one.
3097        if let Some(store) = &self.thread_store
3098            && let Some(snapshot) = store.load_thread(thread_id).await?
3099            && let Some(mut metadata) = snapshot.metadata
3100        {
3101            metadata.runner_destination = Some(destination);
3102            metadata.runner_state = Some(state.clone());
3103            metadata.updated_at = OffsetDateTime::now_utc();
3104            store.update_thread_metadata(metadata).await?;
3105        }
3106        self.evict_runner_session(thread_id).await;
3107        Ok(state)
3108    }
3109
3110    /// Rejoin a previously created sandbox from a thread's persisted runner
3111    /// state without provisioning a new one. An optional `sandbox` overrides the
3112    /// persisted sandbox name (recovery by name). Persists refreshed state.
3113    pub async fn rejoin_thread_runner(
3114        &self,
3115        thread_id: &ThreadId,
3116        sandbox: Option<String>,
3117    ) -> anyhow::Result<RunnerSessionState> {
3118        let store = self
3119            .thread_store
3120            .as_ref()
3121            .ok_or_else(|| anyhow::anyhow!("thread store is required to rejoin a runner"))?;
3122        let metadata = store
3123            .load_thread_metadata(thread_id)
3124            .await?
3125            .ok_or_else(|| anyhow::anyhow!("thread {thread_id} has no metadata"))?;
3126        let destination = metadata
3127            .runner_binding
3128            .as_ref()
3129            .map(|binding| binding.destination.clone())
3130            .or_else(|| metadata.runner_destination.clone())
3131            .ok_or_else(|| anyhow::anyhow!("thread {thread_id} is not bound to a remote runner"))?;
3132        let mut state = metadata
3133            .runner_state
3134            .clone()
3135            .ok_or_else(|| anyhow::anyhow!("thread {thread_id} has no persisted runner state"))?;
3136        if let Some(sandbox) = sandbox
3137            && let Some(object) = state.metadata.as_object_mut()
3138        {
3139            object.insert("sandbox_name".to_string(), serde_json::Value::from(sandbox));
3140        }
3141        let provider = self
3142            .remote_runner_provider_by_id(&destination.provider_id)
3143            .ok_or_else(|| {
3144                anyhow::anyhow!(
3145                    "remote runner provider {:?} is not installed",
3146                    destination.provider_id
3147                )
3148            })?;
3149        let session = provider.rejoin_session(state).await?;
3150        self.persist_runner_state(thread_id, Some(&(destination.clone(), session.clone())))
3151            .await?;
3152        self.cache_runner_session(thread_id, destination, session.clone())
3153            .await;
3154        Ok(session.state())
3155    }
3156
3157    pub(crate) async fn record_thread_usage_metadata(
3158        &self,
3159        thread_id: &ThreadId,
3160        usage: &TokenUsage,
3161    ) -> anyhow::Result<()> {
3162        if usage.is_empty() {
3163            return Ok(());
3164        }
3165        let _guard = self.thread_usage_metadata_lock.lock().await;
3166        let Some(store) = &self.thread_store else {
3167            return Ok(());
3168        };
3169        let Some(snapshot) = store.load_thread(thread_id).await? else {
3170            return Ok(());
3171        };
3172        let Some(mut metadata) = snapshot.metadata else {
3173            return Ok(());
3174        };
3175        metadata
3176            .usage
3177            .get_or_insert_with(ThreadUsageMetadata::default)
3178            .add_token_usage(usage);
3179        metadata.updated_at = OffsetDateTime::now_utc();
3180        store.update_thread_metadata(metadata).await?;
3181        Ok(())
3182    }
3183
3184    pub(crate) async fn has_active_turn_for_thread(&self, thread_id: &ThreadId) -> bool {
3185        self.active_turns
3186            .read()
3187            .await
3188            .values()
3189            .any(|handle| &handle.thread_id == thread_id)
3190    }
3191
3192    async fn ensure_codex_v2_team_capacity(
3193        &self,
3194        team: &TeamState,
3195        resuming_member_id: &str,
3196        candidate_model: Option<&str>,
3197    ) -> anyhow::Result<()> {
3198        if !is_codex_v2_team(team)
3199            && !candidate_model
3200                .is_some_and(|model| model_supports_reasoning_effort(model, REASONING_ULTRA))
3201        {
3202            return Ok(());
3203        }
3204        let active_thread_ids = self
3205            .active_turns
3206            .read()
3207            .await
3208            .values()
3209            .map(|handle| handle.thread_id.clone())
3210            .collect::<std::collections::HashSet<_>>();
3211        let resident_threads = 1 + team
3212            .members
3213            .iter()
3214            .filter(|member| {
3215                member.role != roder_api::teams::TeamMemberRole::Lead
3216                    && member.id != resuming_member_id
3217                    && active_thread_ids.contains(&member.thread_id)
3218            })
3219            .count();
3220        anyhow::ensure!(
3221            resident_threads < codex_v2::CODEX_V2_MAX_RESIDENT_TEAM_THREADS,
3222            "agent thread limit reached for this Codex V2 team: maximum {} resident threads (the lead plus 3 running subagents)",
3223            codex_v2::CODEX_V2_MAX_RESIDENT_TEAM_THREADS
3224        );
3225        Ok(())
3226    }
3227
3228    /// Number of currently running turns across all threads. Hosted runtime
3229    /// pools use this to avoid evicting tenants with active work.
3230    pub async fn active_turn_count(&self) -> usize {
3231        self.active_turns.read().await.len()
3232    }
3233
3234    pub async fn active_turn_for_thread(&self, thread_id: &ThreadId) -> Option<TurnId> {
3235        self.active_turns
3236            .read()
3237            .await
3238            .iter()
3239            .find_map(|(turn_id, handle)| (&handle.thread_id == thread_id).then(|| turn_id.clone()))
3240    }
3241
3242    pub async fn thread_activity(&self, thread_id: &ThreadId) -> ThreadActivity {
3243        let active_turn_id = self.active_turn_for_thread(thread_id).await;
3244        let draining_turn_id = if active_turn_id.is_none() {
3245            self.turn_drains
3246                .read()
3247                .await
3248                .iter()
3249                .find_map(|(turn_id, drain)| {
3250                    (&drain.thread_id == thread_id).then(|| turn_id.clone())
3251                })
3252        } else {
3253            None
3254        };
3255        let Some(active_turn_id) = active_turn_id.or(draining_turn_id) else {
3256            return ThreadActivity::default();
3257        };
3258
3259        let mut active_flags = Vec::new();
3260        {
3261            let pending_approvals = self.pending_tool_approvals.lock().await;
3262            if pending_approvals
3263                .values()
3264                .any(|pending| &pending.thread_id == thread_id && pending.turn_id == active_turn_id)
3265            {
3266                active_flags.push("approvalRequired".to_string());
3267            }
3268        }
3269        {
3270            let pending_inputs = self.pending_user_inputs.lock().await;
3271            if pending_inputs
3272                .values()
3273                .any(|pending| &pending.thread_id == thread_id && pending.turn_id == active_turn_id)
3274            {
3275                active_flags.push("userInputRequired".to_string());
3276            }
3277        }
3278        {
3279            let pending_external = self.pending_external_tool_calls.lock().await;
3280            if pending_external
3281                .values()
3282                .any(|pending| &pending.thread_id == thread_id && pending.turn_id == active_turn_id)
3283            {
3284                active_flags.push("externalToolPending".to_string());
3285            }
3286        }
3287        let interrupting = self
3288            .active_turns
3289            .read()
3290            .await
3291            .get(&active_turn_id)
3292            .is_some_and(|handle| handle.drain.interrupt_requested.load(Ordering::Acquire))
3293            || self
3294                .turn_drains
3295                .read()
3296                .await
3297                .get(&active_turn_id)
3298                .is_some_and(|drain| drain.interrupt_requested.load(Ordering::Acquire));
3299        if interrupting {
3300            active_flags.push("interrupting".to_string());
3301        }
3302        if self.pending_plan_exit().await.is_some_and(|pending| {
3303            &pending.thread_id == thread_id && pending.turn_id == active_turn_id
3304        }) {
3305            active_flags.push("planExitRequired".to_string());
3306        }
3307
3308        ThreadActivity {
3309            active_turn_id: Some(active_turn_id),
3310            active_flags,
3311        }
3312    }
3313
3314    pub async fn interrupt_turn(&self, thread_id: ThreadId, turn_id: TurnId) -> anyhow::Result<()> {
3315        self.interrupt_turn_with_reason(thread_id, turn_id, TurnLifecycleReason::UserInterrupt)
3316            .await
3317    }
3318
3319    async fn interrupt_turn_with_reason(
3320        &self,
3321        thread_id: ThreadId,
3322        turn_id: TurnId,
3323        reason: TurnLifecycleReason,
3324    ) -> anyhow::Result<()> {
3325        let handle = self.active_turns.write().await.remove(&turn_id);
3326        if let Some(handle) = handle {
3327            if handle
3328                .drain
3329                .interrupt_requested
3330                .swap(true, Ordering::AcqRel)
3331            {
3332                return Ok(());
3333            }
3334            *handle.drain.interrupt_reason.lock().await = Some(reason);
3335            self.turn_drains
3336                .write()
3337                .await
3338                .insert(turn_id.clone(), handle.drain.clone());
3339            let ownership = self.provider_cleanup_ownership(&turn_id);
3340            self.record_turn_lifecycle_with_ownership(
3341                thread_id.clone(),
3342                turn_id.clone(),
3343                TurnLifecycleState::InterruptRequested,
3344                TurnCleanupState::Requested,
3345                Some(reason),
3346                ownership,
3347            )
3348            .await;
3349            handle.abort.abort();
3350            self.sync_turn_goal_usage(&thread_id, &turn_id).await?;
3351            if reason == TurnLifecycleReason::UserInterrupt {
3352                self.thread_events.remove_thread(&thread_id).await;
3353                self.goals
3354                    .finish_turn(
3355                        &thread_id,
3356                        &turn_id,
3357                        Some(roder_api::goals::ThreadGoalStatus::Paused),
3358                    )
3359                    .await?;
3360            }
3361            self.active_turns_changed.notify_waiters();
3362        }
3363        self.active_turn_selections.write().await.remove(&turn_id);
3364        self.cancel_pending_external_tool_calls_for_turn(&turn_id)
3365            .await;
3366        Ok(())
3367    }
3368
3369    pub async fn steer_turn(
3370        &self,
3371        thread_id: ThreadId,
3372        turn_id: TurnId,
3373        message: String,
3374        images: Vec<InputImage>,
3375    ) -> anyhow::Result<()> {
3376        self.enqueue_turn_steer(thread_id, turn_id, message, images, None)
3377            .await
3378    }
3379
3380    pub(crate) async fn has_pending_turn_steers(&self, turn_id: &TurnId) -> bool {
3381        let active = self.active_turns.read().await.get(turn_id).cloned();
3382        let Some(active) = active else {
3383            return false;
3384        };
3385        let has_pending = !active.steers.lock().await.is_empty();
3386        has_pending
3387    }
3388
3389    async fn steer_turn_with_mailbox_ack(
3390        &self,
3391        thread_id: ThreadId,
3392        turn_id: TurnId,
3393        message: String,
3394        message_ids: Vec<String>,
3395        team_id: TeamId,
3396    ) -> anyhow::Result<()> {
3397        self.enqueue_turn_steer(
3398            thread_id,
3399            turn_id,
3400            message,
3401            Vec::new(),
3402            Some(MailboxDeliveryAck {
3403                team_id,
3404                message_ids,
3405            }),
3406        )
3407        .await
3408    }
3409
3410    async fn deliver_pending_team_mailbox(
3411        &self,
3412        team_id: &str,
3413        member_id: &str,
3414        member_thread_id: ThreadId,
3415        turn_id: TurnId,
3416    ) -> anyhow::Result<()> {
3417        let pending = self
3418            .teams
3419            .reserve_pending_mailbox_messages(team_id, member_id, &turn_id)
3420            .await?;
3421        if pending.is_empty() {
3422            return Ok(());
3423        }
3424        let message_ids = pending
3425            .iter()
3426            .map(|message| message.id.clone())
3427            .collect::<Vec<_>>();
3428        let team = self
3429            .read_team(team_id)
3430            .await
3431            .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
3432        if let Err(error) = self
3433            .steer_turn_with_mailbox_ack(
3434                member_thread_id,
3435                turn_id.clone(),
3436                format_mailbox_messages(&team, &pending),
3437                message_ids.clone(),
3438                team_id.to_string(),
3439            )
3440            .await
3441        {
3442            self.teams
3443                .release_mailbox_reservations(&turn_id, &message_ids)
3444                .await;
3445            return Err(error);
3446        }
3447        Ok(())
3448    }
3449
3450    async fn enqueue_turn_steer(
3451        &self,
3452        thread_id: ThreadId,
3453        turn_id: TurnId,
3454        message: String,
3455        images: Vec<InputImage>,
3456        mailbox_ack: Option<MailboxDeliveryAck>,
3457    ) -> anyhow::Result<()> {
3458        let message = message.trim().to_string();
3459        if message.is_empty() && images.is_empty() {
3460            return Ok(());
3461        }
3462
3463        let Some(active) = self.active_turns.read().await.get(&turn_id).cloned() else {
3464            anyhow::bail!("no active turn to steer");
3465        };
3466        anyhow::ensure!(
3467            active.thread_id == thread_id,
3468            "turn does not belong to thread"
3469        );
3470        {
3471            let mut steers = active.steers.lock().await;
3472            steers.push(QueuedTurnSteer {
3473                message: UserMessage::with_images(message.clone(), images),
3474                mailbox_ack,
3475            });
3476            active
3477                .steer_changed
3478                .send_modify(|generation| *generation = generation.wrapping_add(1));
3479        }
3480        self.emit(RoderEvent::TurnSteered(TurnSteered {
3481            thread_id,
3482            turn_id,
3483            message,
3484            timestamp: OffsetDateTime::now_utc(),
3485        }))
3486        .await;
3487        Ok(())
3488    }
3489
3490    pub async fn tool_specs(&self) -> Vec<roder_api::tools::ToolSpec> {
3491        let cfg = self.config.read().await;
3492        let model_profile =
3493            model_profile_for_provider_model(&cfg, &cfg.default_provider, &cfg.default_model);
3494        self.filtered_tool_specs(
3495            &cfg,
3496            &cfg.default_model,
3497            &cfg.default_provider,
3498            model_profile.as_ref(),
3499            &[],
3500            &[],
3501        )
3502    }
3503
3504    pub fn subagent_definitions(&self) -> Vec<SubagentDefinition> {
3505        self.registry
3506            .subagent_dispatchers
3507            .iter()
3508            .flat_map(|dispatcher| dispatcher.definitions())
3509            .collect()
3510    }
3511
3512    async fn run_turn(
3513        self: &Arc<Self>,
3514        req: StartTurnRequest,
3515        turn_id: TurnId,
3516        initial_mailbox_ack: Option<MailboxDeliveryAck>,
3517        mut steering: tokio::sync::watch::Receiver<u64>,
3518        automatic_goal_turn: bool,
3519    ) -> anyhow::Result<TurnRunOutcome> {
3520        self.ensure_execution_authority()?;
3521        self.goals
3522            .begin_turn(&req.thread_id, &turn_id, automatic_goal_turn)
3523            .await?;
3524        let turn_started_at = OffsetDateTime::now_utc();
3525        self.emit(RoderEvent::TurnStarted(TurnStarted {
3526            thread_id: req.thread_id.clone(),
3527            turn_id: turn_id.clone(),
3528            runtime_profile: self.config.read().await.runtime_profile,
3529            timestamp: turn_started_at,
3530        }))
3531        .await;
3532        self.persist_turn_item(
3533            &req.thread_id,
3534            &turn_id,
3535            &TranscriptItem::UserMessage(UserMessage::with_images(
3536                req.message.clone(),
3537                req.images.clone(),
3538            )),
3539        )
3540        .await?;
3541        crate::hooks::run_lifecycle(
3542            self,
3543            &req.thread_id,
3544            &turn_id,
3545            Some(&req.workspace),
3546            "UserPromptSubmit",
3547            None,
3548            serde_json::json!({"prompt": req.message}),
3549        )
3550        .await;
3551        if let Some(ack) = initial_mailbox_ack {
3552            self.teams
3553                .mark_mailbox_messages_delivered(&ack.team_id, &turn_id, &ack.message_ids)
3554                .await?;
3555        }
3556
3557        let mut cfg = self.config.read().await.clone();
3558        let runtime_profile = cfg.runtime_profile;
3559        let turn_deadline = turn_deadline_for_config(&cfg);
3560        let deadline_finalization_reserve =
3561            crate::deadline_policy::finalization_reserve_seconds(cfg.turn_deadline_seconds);
3562        let selection_mode = self.selection_mode_for_thread(&req.thread_id).await?;
3563        let concrete_selection = selection_mode
3564            .as_ref()
3565            .map(ModelSelectionMode::concrete_selection);
3566        let default_provider = req
3567            .provider_override
3568            .clone()
3569            .or_else(|| {
3570                concrete_selection
3571                    .as_ref()
3572                    .map(|selection| selection.provider.clone())
3573            })
3574            .unwrap_or(cfg.default_provider.clone());
3575        let default_model = req
3576            .model_override
3577            .clone()
3578            .or_else(|| {
3579                concrete_selection
3580                    .as_ref()
3581                    .map(|selection| selection.model.clone())
3582            })
3583            .unwrap_or(cfg.default_model.clone());
3584        if let Some(reasoning) = req.reasoning_override.as_deref().or_else(|| {
3585            selection_mode
3586                .as_ref()
3587                .and_then(ModelSelectionMode::reasoning)
3588        }) {
3589            validate_reasoning_effort(&default_model, reasoning)?;
3590            cfg.reasoning = Some(reasoning.to_string());
3591        }
3592        let turn_has_concrete_model_override =
3593            req.provider_override.is_some() || req.model_override.is_some();
3594        let (turn_inference_router, turn_inference_router_profile) = match &selection_mode {
3595            Some(ModelSelectionMode::Auto {
3596                router_id, profile, ..
3597            }) if !turn_has_concrete_model_override => (
3598                RuntimeInferenceRouterConfig {
3599                    enabled: true,
3600                    router_id: Some(router_id.clone()),
3601                },
3602                profile.clone(),
3603            ),
3604            _ => (RuntimeInferenceRouterConfig::disabled(), None),
3605        };
3606        let mut provider = default_provider.clone();
3607        let mut model = default_model.clone();
3608        let mut model_profile = model_profile_for_provider_model(&cfg, &provider, &model);
3609        let workspace = req.workspace.clone();
3610        let compaction_generation_at_start = self.compaction_generation(&req.thread_id);
3611        let mut transcript = self.transcript_for_turn(&req, &turn_id).await?;
3612        let mut compacted_this_turn =
3613            self.compaction_generation(&req.thread_id) != compaction_generation_at_start;
3614        let agent_swarm_mode_active = self
3615            .effective_agent_swarm_mode_for_thread(&req.thread_id)
3616            .await;
3617        let ultra_mode_active = self.effective_ultra_mode_for_thread(&req.thread_id).await;
3618        let thread_overrides = self.thread_turn_overrides(&req.thread_id).await?;
3619        let mut final_assistant_text = String::new();
3620        let mut final_provider_metadata = None;
3621        let mut exhausted_tool_rounds = true;
3622        let mut verification_gate =
3623            VerificationGateState::new(req.message.clone(), runtime_profile);
3624        let mut speed_policy = SpeedPolicyState::default();
3625        let mut reliability = TurnReliabilityState::default();
3626        // Overwritten on every inference step's Completed event so only the
3627        // terminal step's stop reason survives (mid-turn tool_use steps must
3628        // not leak onto turn/completed).
3629        let mut turn_finish_reason: Option<String> = None;
3630        let mut deadline_finalization_requested = false;
3631        let mut deadline_scoreable_completion_requested = false;
3632        let mut task_ledger_completion_reminders = 0_u8;
3633        let mut task_ledger_scoreable_checkpoints = 0_u8;
3634        let mut empty_tool_call_nudges_used = 0_u32;
3635        let mut provider_stream_retry_attempts = 0_u32;
3636        let mut context_limit_recovery_attempts = 0_u32;
3637        let mut routing_candidates = None;
3638        let routing_transcript_start = transcript.len().saturating_sub(1);
3639        let mut routing_escalations = 0_u32;
3640        let mut model_switch_summary_selection = None::<ModelSelection>;
3641
3642        'tool_rounds: for round_index in 0..MAX_TOOL_ROUNDS_PER_TURN {
3643            if let Some(deadline) = turn_deadline
3644                && deadline_expired(deadline)
3645            {
3646                self.fail_turn_due_to_deadline(&req.thread_id, &turn_id, deadline, &transcript)
3647                    .await?;
3648                return Ok(TurnRunOutcome::Stopped);
3649            }
3650            let steers = self.drain_turn_steers(&turn_id, &mut steering).await;
3651            self.append_steers(&req, &turn_id, &mut transcript, steers)
3652                .await?;
3653            if runtime_profile == RuntimeProfile::Eval
3654                && let Some(remaining) = crate::deadline_policy::should_start_finalization(
3655                    turn_deadline,
3656                    deadline_finalization_reserve,
3657                    deadline_finalization_requested || deadline_scoreable_completion_requested,
3658                )
3659            {
3660                if req.task_ledger_required
3661                    && task_ledger_completion_reminders < TASK_LEDGER_COMPLETION_REMINDER_LIMIT
3662                    && let Some(prompt) = task_ledger_completion_prompt(&transcript)
3663                {
3664                    task_ledger_completion_reminders += 1;
3665                    deadline_scoreable_completion_requested = true;
3666                    let item = TranscriptItem::UserMessage(UserMessage::text(
3667                        task_ledger_deadline_completion_prompt(
3668                            remaining,
3669                            deadline_finalization_reserve,
3670                            &prompt,
3671                        ),
3672                    ));
3673                    self.persist_turn_item(&req.thread_id, &turn_id, &item)
3674                        .await?;
3675                    transcript.push(item);
3676                    continue 'tool_rounds;
3677                } else {
3678                    self.start_deadline_finalization(
3679                        &req.thread_id,
3680                        &turn_id,
3681                        &mut transcript,
3682                        remaining,
3683                    )
3684                    .await?;
3685                    deadline_finalization_requested = true;
3686                }
3687            }
3688            if runtime_profile == RuntimeProfile::Eval
3689                && req.task_ledger_required
3690                && !deadline_finalization_requested
3691                && task_ledger_scoreable_checkpoints < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
3692                && let Some(remaining) = deadline_remaining_seconds(turn_deadline)
3693                && remaining <= TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS
3694                && remaining > deadline_finalization_reserve
3695                && let Some(prompt) = task_ledger_completion_prompt(&transcript)
3696            {
3697                task_ledger_scoreable_checkpoints += 1;
3698                let item = TranscriptItem::UserMessage(UserMessage::text(
3699                    task_ledger_scoreable_checkpoint_prompt(remaining, &prompt),
3700                ));
3701                self.persist_turn_item(&req.thread_id, &turn_id, &item)
3702                    .await?;
3703                transcript.push(item);
3704                continue 'tool_rounds;
3705            }
3706            if turn_inference_router.is_active() && routing_candidates.is_none() {
3707                routing_candidates =
3708                    Some(collect_inference_routing_candidates(&self.registry).await);
3709            }
3710            let routing_tools_model = model.clone();
3711            let routing_tools_provider = provider.clone();
3712            let routing_tools = self.filtered_tool_specs(
3713                &cfg,
3714                &model,
3715                &provider,
3716                model_profile.as_ref(),
3717                &thread_overrides.tool_allowlist,
3718                &thread_overrides.external_tools,
3719            );
3720            let prior_failures =
3721                transcript_failure_count_since(&transcript, routing_transcript_start)
3722                    .max(reliability.tool_failure_count())
3723                    .saturating_add(provider_stream_retry_attempts);
3724            let routing_selection = route_inference_selection(
3725                &self.registry,
3726                &turn_inference_router,
3727                InferenceRoutingRequest {
3728                    thread_id: &req.thread_id,
3729                    turn_id: &turn_id,
3730                    round_index: round_index as u32,
3731                    runtime_profile,
3732                    phase: speed_policy.phase(),
3733                    profile: turn_inference_router_profile.as_deref(),
3734                    default_selection: ModelSelection {
3735                        provider: default_provider.clone(),
3736                        model: default_model.clone(),
3737                    },
3738                    transcript: &transcript,
3739                    tools: &routing_tools,
3740                    candidates: routing_candidates.as_deref(),
3741                    prior_failures,
3742                    prior_escalations: routing_escalations,
3743                },
3744            )
3745            .await;
3746            if let Some(decision) = routing_selection.decision.clone() {
3747                if matches!(decision.outcome, InferenceRoutingOutcome::Escalated) {
3748                    routing_escalations = routing_escalations.saturating_add(1);
3749                }
3750                self.emit(RoderEvent::InferenceRoutingDecision(
3751                    InferenceRoutingDecisionEvent {
3752                        thread_id: req.thread_id.clone(),
3753                        turn_id: turn_id.clone(),
3754                        round_index: round_index as u32,
3755                        default_selection: ModelSelection {
3756                            provider: default_provider.clone(),
3757                            model: default_model.clone(),
3758                        },
3759                        selected_selection: routing_selection.selection.clone(),
3760                        decision,
3761                        timestamp: OffsetDateTime::now_utc(),
3762                    },
3763                ))
3764                .await;
3765            }
3766            provider = routing_selection.selection.provider.clone();
3767            model = routing_selection.selection.model.clone();
3768            let engine = self.engine_for(&provider)?;
3769            let capabilities = engine.capabilities();
3770            let tool_images = ToolImages::for_engine(engine.as_ref(), &model);
3771            model_profile = model_profile_for_provider_model(&cfg, &provider, &model);
3772            let tools = if capabilities.tool_calls {
3773                if model == routing_tools_model && provider == routing_tools_provider {
3774                    routing_tools.clone()
3775                } else {
3776                    self.filtered_tool_specs(
3777                        &cfg,
3778                        &model,
3779                        &provider,
3780                        model_profile.as_ref(),
3781                        &thread_overrides.tool_allowlist,
3782                        &thread_overrides.external_tools,
3783                    )
3784                }
3785            } else {
3786                Vec::new()
3787            };
3788            let parallel_tool_calls = parallel_tool_calls_for_model(&cfg, &model);
3789            let tool_choice = if tools.is_empty() {
3790                ToolChoice::None
3791            } else {
3792                ToolChoice::Auto
3793            };
3794            let summary_selection = ModelSelection {
3795                provider: provider.clone(),
3796                model: model.clone(),
3797            };
3798            if model_switch_summary_selection.as_ref() != Some(&summary_selection) {
3799                if let Some(summary) = model_switch_summary(
3800                    &transcript,
3801                    model_profile.as_ref(),
3802                    &provider,
3803                    &model,
3804                    &tools,
3805                ) {
3806                    let item = TranscriptItem::UserMessage(UserMessage::text(summary));
3807                    self.persist_turn_item(&req.thread_id, &turn_id, &item)
3808                        .await?;
3809                    transcript.push(item);
3810                }
3811                model_switch_summary_selection = Some(summary_selection);
3812            }
3813
3814            if !capabilities.image_input && transcript_has_images(&transcript) {
3815                self.fail_turn_with_error(
3816                    &req.thread_id,
3817                    &turn_id,
3818                    format!("provider {provider} does not support image input"),
3819                )
3820                .await?;
3821                return Ok(TurnRunOutcome::Stopped);
3822            }
3823            let compaction_generation_before = self.compaction_generation(&req.thread_id);
3824            transcript = self
3825                .compact_transcript_if_needed(
3826                    &req.thread_id,
3827                    &turn_id,
3828                    &provider,
3829                    &model,
3830                    transcript,
3831                    self.compaction_options_for_turn(&req.thread_id, !compacted_this_turn),
3832                )
3833                .await?;
3834            compacted_this_turn |=
3835                self.compaction_generation(&req.thread_id) != compaction_generation_before;
3836            if round_index == 0 {
3837                self.complete_context_assembly(&req, &turn_id, &transcript, tool_images)
3838                    .await;
3839            }
3840
3841            let speed_policy_decision =
3842                speed_policy.decision(runtime_profile, &model, &cfg.speed_policy);
3843            let request_reasoning = reasoning_from_decision(
3844                speed_policy_decision.as_ref(),
3845                routing_selection
3846                    .reasoning
3847                    .clone()
3848                    .unwrap_or_else(|| reasoning_for_model(&cfg, &model)),
3849            );
3850            self.active_turn_selections.write().await.insert(
3851                turn_id.clone(),
3852                ModelSelectionMode::manual(
3853                    provider.clone(),
3854                    model.clone(),
3855                    request_reasoning.level.clone(),
3856                ),
3857            );
3858            if let Some(limit) = reliability.record_model_call(
3859                &cfg.reliability,
3860                runtime_profile == RuntimeProfile::Interactive,
3861            ) {
3862                self.fail_turn_due_to_reliability_limit(
3863                    &req.thread_id,
3864                    &turn_id,
3865                    &provider,
3866                    &model,
3867                    limit,
3868                    &transcript,
3869                )
3870                .await?;
3871                return Ok(TurnRunOutcome::Stopped);
3872            }
3873            self.emit(RoderEvent::InferenceStarted(InferenceStarted {
3874                thread_id: req.thread_id.clone(),
3875                turn_id: turn_id.clone(),
3876                engine_id: engine.id(),
3877                model: ModelSelection {
3878                    provider: provider.clone(),
3879                    model: model.clone(),
3880                },
3881                reasoning: request_reasoning.clone(),
3882                speed_policy: speed_policy_decision.clone(),
3883                deadline_remaining_seconds: deadline_remaining_seconds(turn_deadline),
3884                timestamp: OffsetDateTime::now_utc(),
3885            }))
3886            .await;
3887
3888            let mut instructions = req.instructions.clone();
3889            if let Some(extra) = &thread_overrides.developer_instructions {
3890                instructions = apply_thread_developer_instructions(instructions, extra);
3891            }
3892            if let Some(context) = req.developer_context.as_deref() {
3893                instructions = apply_turn_developer_context(instructions, context);
3894            }
3895            let mut instructions = apply_runtime_profile(instructions, runtime_profile);
3896            if let Some(profile) = &model_profile {
3897                instructions = apply_model_instruction_overlay(instructions, profile);
3898            }
3899            if req.task_ledger_required
3900                && runtime_profile == RuntimeProfile::Eval
3901                && !transcript_has_task_ledger(&transcript)
3902            {
3903                instructions = apply_task_ledger_required(instructions);
3904            }
3905            let effective_policy_mode = self.effective_policy_mode_for_thread(&req.thread_id).await;
3906            if effective_policy_mode == PolicyMode::Plan {
3907                instructions = apply_plan_mode(instructions);
3908            }
3909            if agent_swarm_mode_active {
3910                instructions = apply_agent_swarm_mode(instructions);
3911            }
3912            // Ultra mode (any model) enables proactive multi-agent policy.
3913            // Codex Sol/Terra Ultra effort still does too without requiring the
3914            // mode flag. Models that advertise Ultra effort keep the
3915            // explicit-request-only policy on lower efforts when ultra mode is
3916            // off; other models get multi-agent policy only when ultra mode is on.
3917            let model_has_ultra_effort = model_supports_reasoning_effort(&model, REASONING_ULTRA);
3918            let effort_is_ultra = request_reasoning.level.as_deref() == Some(REASONING_ULTRA);
3919            let proactive_multi_agent = ultra_mode_active || effort_is_ultra;
3920            if ultra_mode_active || model_has_ultra_effort {
3921                instructions = apply_codex_multi_agent_mode(instructions, proactive_multi_agent);
3922            }
3923            let compaction_instructions = instructions.clone();
3924            instructions = self
3925                .goals
3926                .apply_goal_instructions(&req.thread_id, instructions, effective_policy_mode)
3927                .await?;
3928            let mut request_metadata = serde_json::json!({});
3929            if let Some(decision) = &speed_policy_decision {
3930                request_metadata["speedPolicy"] = serde_json::json!(decision);
3931            }
3932            if let Some(decision) = routing_selection.decision.as_ref() {
3933                request_metadata["inferenceRouting"] = serde_json::json!(decision);
3934            }
3935            if let Some(remaining) = deadline_remaining_seconds(turn_deadline) {
3936                request_metadata["deadlineRemainingSeconds"] = serde_json::json!(remaining);
3937            }
3938            if let Some(profile) = &model_profile {
3939                request_metadata["modelProfile"] = serde_json::json!({
3940                    "model": profile.model,
3941                    "providerFamily": profile.provider_family,
3942                    "editTool": profile.edit_tool,
3943                    "schemaPolicy": profile.schema_policy,
3944                    "instructionOverlay": profile.instruction_overlay,
3945                    "parallelToolCalls": profile.parallel_tool_calls,
3946                    "autoCompactTokenLimit": profile.auto_compact_token_limit,
3947                });
3948            }
3949            let task_ledger_required_this_round = req.task_ledger_required
3950                && runtime_profile == RuntimeProfile::Eval
3951                && !deadline_finalization_requested
3952                && !transcript_has_task_ledger(&transcript);
3953            let task_ledger_tools = (capabilities.tool_calls && task_ledger_required_this_round)
3954                .then(|| self.task_ledger_tool_specs(model_profile.as_ref()))
3955                .filter(|tools| !tools.is_empty());
3956            let request_tools = if deadline_finalization_requested {
3957                Vec::new()
3958            } else if let Some(ledger_tools) = &task_ledger_tools {
3959                ledger_tools.clone()
3960            } else {
3961                tools.clone()
3962            };
3963            // Tool guidance describes the tools this request can call, which is
3964            // fewer than `tools` on a deadline finalization or forced ledger round.
3965            instructions = apply_parallel_web_tools(
3966                instructions,
3967                request_tools.iter().map(|spec| spec.name.as_str()),
3968            );
3969            instructions = apply_browser_routing(
3970                instructions,
3971                request_tools.iter().map(|spec| spec.name.as_str()),
3972            );
3973            let request_tool_choice = if deadline_finalization_requested {
3974                ToolChoice::None
3975            } else if task_ledger_tools.is_some() {
3976                ToolChoice::Specific(TASK_LEDGER_TOOL_NAME.to_string())
3977            } else {
3978                tool_choice.clone()
3979            };
3980            if deadline_finalization_requested {
3981                request_metadata["deadlineFinalization"] = serde_json::json!({
3982                    "reserveSeconds": deadline_finalization_reserve,
3983                    "remainingSeconds": deadline_remaining_seconds(turn_deadline),
3984                });
3985            } else if deadline_scoreable_completion_requested {
3986                request_metadata["deadlineScoreableCompletion"] = serde_json::json!({
3987                    "reserveSeconds": deadline_finalization_reserve,
3988                    "remainingSeconds": deadline_remaining_seconds(turn_deadline),
3989                });
3990            }
3991            let mut eager_tools = EagerTools::new(
3992                self,
3993                &request_tools,
3994                &thread_overrides.external_tools,
3995                parallel_tool_calls,
3996            );
3997            let mut request_reliability: ReliabilityRequestPolicy = cfg.reliability.clone().into();
3998            request_reliability.provider_retry_max_attempts = request_reliability
3999                .provider_retry_max_attempts
4000                .saturating_sub(provider_stream_retry_attempts)
4001                .max(1);
4002            let request = AgentInferenceRequest {
4003                model: ModelSelection {
4004                    provider: provider.clone(),
4005                    model: model.clone(),
4006                },
4007                instructions,
4008                transcript: crate::tool_result_images::request_transcript(&transcript, tool_images),
4009                tools: request_tools,
4010                tool_choice: request_tool_choice,
4011                reasoning: request_reasoning,
4012                output: OutputConfig::default(),
4013                runtime: RuntimeHints {
4014                    auto_compact_token_limit: (provider != roder_api::catalog::PROVIDER_CODEX)
4015                        .then(|| server_side_compaction_threshold(&cfg, &model))
4016                        .flatten(),
4017                    profile: runtime_profile,
4018                    parallel_tool_calls: Some(parallel_tool_calls),
4019                    prompt_cache_key: Some(req.thread_id.clone()),
4020                    hosted_web_search: cfg.hosted_web_search.clone(),
4021                    tool_search: tool_search_for_provider_model(&cfg, &provider, &model),
4022                    speed_policy: speed_policy_decision,
4023                    reliability: Some(request_reliability),
4024                    deadline_remaining_seconds: deadline_remaining_seconds(turn_deadline),
4025                    service_tier: req
4026                        .service_tier_override
4027                        .clone()
4028                        .or_else(|| thread_overrides.service_tier.clone()),
4029                    ..RuntimeHints::default()
4030                },
4031                metadata: request_metadata,
4032            };
4033
4034            let mut compaction_template = request.clone();
4035            compaction_template.instructions = compaction_instructions;
4036            compaction_template.transcript.clear();
4037            self.compaction_templates
4038                .write()
4039                .await
4040                .insert(req.thread_id.clone(), compaction_template);
4041
4042            let ctx = InferenceTurnContext {
4043                thread_id: &req.thread_id,
4044                turn_id: &turn_id,
4045                tool_executor: Some(std::sync::Arc::new(
4046                    crate::tool_execution::RuntimeTurnToolExecutor {
4047                        runtime: Arc::clone(self),
4048                        thread_id: req.thread_id.clone(),
4049                        turn_id: turn_id.clone(),
4050                        workspace: Some(workspace.clone()),
4051                        deadline: turn_deadline,
4052                    },
4053                )),
4054            };
4055            let stream_future = preemptible(engine.stream_turn(ctx, request), &mut steering);
4056            let mut stream = if let Some((deadline, timeout_action)) = inference_timeout_deadline(
4057                turn_deadline,
4058                runtime_profile,
4059                req.task_ledger_required,
4060                deadline_finalization_reserve,
4061                deadline_finalization_requested || deadline_scoreable_completion_requested,
4062                task_ledger_scoreable_checkpoints,
4063                &transcript,
4064            ) {
4065                match tokio::time::timeout_at(deadline_instant(deadline), stream_future).await {
4066                    Ok(stream) => match stream {
4067                        Ok(stream) => stream,
4068                        Err(err) => {
4069                            if err.is::<SamplingPreempted>() {
4070                                self.finish_sampling_cleanup(&turn_id).await?;
4071                                provider_stream_retry_attempts = 0;
4072                                continue 'tool_rounds;
4073                            }
4074                            self.finish_sampling_cleanup(&turn_id).await?;
4075                            if !deadline_finalization_requested
4076                                && self
4077                                    .retry_sampling_failure(
4078                                        SamplingRetry {
4079                                            thread_id: &req.thread_id,
4080                                            turn_id: &turn_id,
4081                                            provider: &provider,
4082                                            model: &model,
4083                                            config: &cfg.reliability,
4084                                            error: &err,
4085                                        },
4086                                        &mut provider_stream_retry_attempts,
4087                                        &mut steering,
4088                                    )
4089                                    .await
4090                            {
4091                                continue 'tool_rounds;
4092                            }
4093                            let error = err.to_string();
4094                            if self
4095                                .try_recover_from_context_limit(
4096                                    &req.thread_id,
4097                                    &turn_id,
4098                                    &provider,
4099                                    &model,
4100                                    &error,
4101                                    crate::compaction::is_context_overflow_failure(&err),
4102                                    &mut transcript,
4103                                    &mut compacted_this_turn,
4104                                    &mut context_limit_recovery_attempts,
4105                                )
4106                                .await?
4107                            {
4108                                continue 'tool_rounds;
4109                            }
4110                            return Err(err);
4111                        }
4112                    },
4113                    Err(_) => {
4114                        if runtime_profile == RuntimeProfile::Eval
4115                            && !deadline_finalization_requested
4116                        {
4117                            let remaining = deadline_remaining_seconds(turn_deadline).unwrap_or(0);
4118                            if timeout_action == InferenceTimeoutAction::ScoreableCheckpoint
4119                                && task_ledger_scoreable_checkpoints
4120                                    < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
4121                                && let Some(prompt) = task_ledger_completion_prompt(&transcript)
4122                            {
4123                                task_ledger_scoreable_checkpoints += 1;
4124                                let item = TranscriptItem::UserMessage(UserMessage::text(
4125                                    task_ledger_scoreable_checkpoint_prompt(remaining, &prompt),
4126                                ));
4127                                self.persist_turn_item(&req.thread_id, &turn_id, &item)
4128                                    .await?;
4129                                transcript.push(item);
4130                                continue 'tool_rounds;
4131                            }
4132                            self.start_deadline_finalization(
4133                                &req.thread_id,
4134                                &turn_id,
4135                                &mut transcript,
4136                                remaining,
4137                            )
4138                            .await?;
4139                            deadline_finalization_requested = true;
4140                            continue 'tool_rounds;
4141                        }
4142                        self.fail_turn_due_to_deadline(
4143                            &req.thread_id,
4144                            &turn_id,
4145                            deadline,
4146                            &transcript,
4147                        )
4148                        .await?;
4149                        return Ok(TurnRunOutcome::Stopped);
4150                    }
4151                }
4152            } else {
4153                match stream_future.await {
4154                    Ok(stream) => stream,
4155                    Err(err) => {
4156                        if err.is::<SamplingPreempted>() {
4157                            self.finish_sampling_cleanup(&turn_id).await?;
4158                            provider_stream_retry_attempts = 0;
4159                            continue 'tool_rounds;
4160                        }
4161                        self.finish_sampling_cleanup(&turn_id).await?;
4162                        if !deadline_finalization_requested
4163                            && self
4164                                .retry_sampling_failure(
4165                                    SamplingRetry {
4166                                        thread_id: &req.thread_id,
4167                                        turn_id: &turn_id,
4168                                        provider: &provider,
4169                                        model: &model,
4170                                        config: &cfg.reliability,
4171                                        error: &err,
4172                                    },
4173                                    &mut provider_stream_retry_attempts,
4174                                    &mut steering,
4175                                )
4176                                .await
4177                        {
4178                            continue 'tool_rounds;
4179                        }
4180                        let error = err.to_string();
4181                        if self
4182                            .try_recover_from_context_limit(
4183                                &req.thread_id,
4184                                &turn_id,
4185                                &provider,
4186                                &model,
4187                                &error,
4188                                crate::compaction::is_context_overflow_failure(&err),
4189                                &mut transcript,
4190                                &mut compacted_this_turn,
4191                                &mut context_limit_recovery_attempts,
4192                            )
4193                            .await?
4194                        {
4195                            continue 'tool_rounds;
4196                        }
4197                        return Err(err);
4198                    }
4199                }
4200            };
4201            let mut sampling_output = SamplingOutput::default();
4202            let output_context = OutputContext {
4203                thread_id: &req.thread_id,
4204                turn_id: &turn_id,
4205                profile: model_profile.as_ref(),
4206                provider: &provider,
4207                model: &model,
4208            };
4209            let mut assistant_text = String::new();
4210            let mut phase_messages = Vec::<AssistantMessage>::new();
4211            let mut reasoning_text = String::new();
4212            let mut replayed_tool_call = false;
4213            let mut tool_calls = Vec::new();
4214            let mut patch_progress = PatchProgressTracker::default();
4215            let mut provider_metadata = None;
4216            let mut provider_requests_continuation = false;
4217            // Captured from mid-stream Compaction(completed) events. Codex/OpenAI
4218            // sometimes abort the SSE stream after emitting the compaction item
4219            // ("error decoding response body") before response.completed, so we
4220            // must not rely solely on ProviderMetadata for the boundary.
4221            let mut stream_compaction_item: Option<serde_json::Value> = None;
4222
4223            loop {
4224                let next_future = async {
4225                    loop {
4226                        tokio::select! {
4227                            biased;
4228                            _ = wait_for_steer(&mut steering) => break Some(Err(SamplingPreempted.into())),
4229                            result = eager_tools.poll_result(), if eager_tools.pending() => {
4230                                if let Err(error) = result { break Some(Err(error)); }
4231                            }
4232                            next = stream.next() => break next,
4233                        }
4234                    }
4235                };
4236                let next = if let Some((deadline, timeout_action)) = inference_timeout_deadline(
4237                    turn_deadline,
4238                    runtime_profile,
4239                    req.task_ledger_required,
4240                    deadline_finalization_reserve,
4241                    deadline_finalization_requested || deadline_scoreable_completion_requested,
4242                    task_ledger_scoreable_checkpoints,
4243                    &transcript,
4244                ) {
4245                    match tokio::time::timeout_at(deadline_instant(deadline), next_future).await {
4246                        Ok(next) => next,
4247                        Err(_) => {
4248                            if runtime_profile == RuntimeProfile::Eval
4249                                && !deadline_finalization_requested
4250                            {
4251                                let remaining =
4252                                    deadline_remaining_seconds(turn_deadline).unwrap_or(0);
4253                                if timeout_action == InferenceTimeoutAction::ScoreableCheckpoint
4254                                    && task_ledger_scoreable_checkpoints
4255                                        < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
4256                                    && let Some(prompt) = task_ledger_completion_prompt(&transcript)
4257                                {
4258                                    task_ledger_scoreable_checkpoints += 1;
4259                                    let item = TranscriptItem::UserMessage(UserMessage::text(
4260                                        task_ledger_scoreable_checkpoint_prompt(remaining, &prompt),
4261                                    ));
4262                                    self.persist_turn_item(&req.thread_id, &turn_id, &item)
4263                                        .await?;
4264                                    transcript.push(item);
4265                                    continue 'tool_rounds;
4266                                }
4267                                self.start_deadline_finalization(
4268                                    &req.thread_id,
4269                                    &turn_id,
4270                                    &mut transcript,
4271                                    remaining,
4272                                )
4273                                .await?;
4274                                deadline_finalization_requested = true;
4275                                continue 'tool_rounds;
4276                            }
4277                            self.fail_turn_due_to_deadline(
4278                                &req.thread_id,
4279                                &turn_id,
4280                                deadline,
4281                                &transcript,
4282                            )
4283                            .await?;
4284                            return Ok(TurnRunOutcome::Stopped);
4285                        }
4286                    }
4287                } else {
4288                    next_future.await
4289                };
4290                let res = next.unwrap_or_else(|| {
4291                    Err(roder_api::provider_error::ProviderFailure::new(
4292                        roder_api::provider_error::ProviderFailureKind::StreamInterrupted,
4293                        "provider stream ended before terminal completion",
4294                    )
4295                    .into())
4296                });
4297                let event = match res {
4298                    Ok(event) => event,
4299                    Err(err) => {
4300                        drop(stream);
4301                        eager_tools.drain().await?;
4302                        for result in eager_tools.take_results(&tool_calls) {
4303                            verification_gate.record_tool_result(&result);
4304                            transcript.push(TranscriptItem::ToolResult(result));
4305                        }
4306                        self.finish_sampling_cleanup(&turn_id).await?;
4307                        if err.is::<SamplingPreempted>() {
4308                            provider_stream_retry_attempts = 0;
4309                            continue 'tool_rounds;
4310                        }
4311                        if !deadline_finalization_requested
4312                            && self
4313                                .retry_sampling_failure(
4314                                    SamplingRetry {
4315                                        thread_id: &req.thread_id,
4316                                        turn_id: &turn_id,
4317                                        provider: &provider,
4318                                        model: &model,
4319                                        config: &cfg.reliability,
4320                                        error: &err,
4321                                    },
4322                                    &mut provider_stream_retry_attempts,
4323                                    &mut steering,
4324                                )
4325                                .await
4326                        {
4327                            continue 'tool_rounds;
4328                        }
4329                        let error = err.to_string();
4330                        if self
4331                            .try_recover_from_context_limit(
4332                                &req.thread_id,
4333                                &turn_id,
4334                                &provider,
4335                                &model,
4336                                &error,
4337                                crate::compaction::is_context_overflow_failure(&err),
4338                                &mut transcript,
4339                                &mut compacted_this_turn,
4340                                &mut context_limit_recovery_attempts,
4341                            )
4342                            .await?
4343                        {
4344                            continue 'tool_rounds;
4345                        }
4346                        return Err(err);
4347                    }
4348                };
4349
4350                let inference_timestamp = OffsetDateTime::now_utc();
4351                self.emit(RoderEvent::InferenceEventReceived(InferenceEventReceived {
4352                    thread_id: req.thread_id.clone(),
4353                    turn_id: turn_id.clone(),
4354                    event: event.clone(),
4355                    timestamp: inference_timestamp,
4356                }))
4357                .await;
4358
4359                match event {
4360                    InferenceEvent::MessageDelta(delta) => {
4361                        if !delta.text.trim().is_empty() {
4362                            self.goals.record_activity(&turn_id).await;
4363                        }
4364                        if let Some((team_id, member)) =
4365                            self.teams.member_for_thread(&req.thread_id).await
4366                        {
4367                            self.emit(RoderEvent::TeamMemberMessageDelta(TeamMemberMessageDelta {
4368                                team_id,
4369                                member_id: member.id,
4370                                member_thread_id: req.thread_id.clone(),
4371                                turn_id: turn_id.clone(),
4372                                delta: delta.text.clone(),
4373                                timestamp: OffsetDateTime::now_utc(),
4374                            }))
4375                            .await;
4376                        }
4377                        if is_final_answer_phase(delta.phase.as_deref()) {
4378                            assistant_text.push_str(&delta.text);
4379                        } else if let Some(last) = phase_messages.last_mut()
4380                            && last.phase == delta.phase
4381                        {
4382                            last.text.push_str(&delta.text);
4383                        } else {
4384                            phase_messages.push(AssistantMessage {
4385                                text: delta.text,
4386                                phase: delta.phase,
4387                            });
4388                        }
4389                    }
4390                    InferenceEvent::ReasoningDelta(delta) => reasoning_text.push_str(&delta.text),
4391                    InferenceEvent::ToolCallCompleted(call) => {
4392                        self.goals.record_activity(&turn_id).await;
4393                        if completed_call_replayed(&transcript, &call)? {
4394                            replayed_tool_call = true;
4395                            continue;
4396                        }
4397                        if let Some(progress) = patch_progress.complete(
4398                            &req.thread_id,
4399                            &turn_id,
4400                            &call.id,
4401                            &call.name,
4402                            &call.arguments,
4403                        ) {
4404                            self.emit(RoderEvent::PatchProgress(progress)).await;
4405                        }
4406                        if !tool_calls
4407                            .iter()
4408                            .any(|previous: &ToolCallCompleted| previous.id == call.id)
4409                        {
4410                            eager_tools
4411                                .schedule(
4412                                    self,
4413                                    &output_context,
4414                                    &call,
4415                                    workspace.as_str(),
4416                                    turn_deadline,
4417                                    &mut transcript,
4418                                )
4419                                .await?;
4420                            tool_calls.push(call);
4421                        }
4422                    }
4423                    InferenceEvent::OutputItemCompleted(item) => {
4424                        sampling_output
4425                            .complete_item(self, &output_context, item, &mut transcript)
4426                            .await?;
4427                    }
4428                    InferenceEvent::Failed(failure) => {
4429                        speed_policy.record_failure();
4430                        let error = failure.message;
4431                        if self
4432                            .try_recover_from_context_limit(
4433                                &req.thread_id,
4434                                &turn_id,
4435                                &provider,
4436                                &model,
4437                                &error,
4438                                false,
4439                                &mut transcript,
4440                                &mut compacted_this_turn,
4441                                &mut context_limit_recovery_attempts,
4442                            )
4443                            .await?
4444                        {
4445                            continue 'tool_rounds;
4446                        }
4447                        self.persist_turn_item(
4448                            &req.thread_id,
4449                            &turn_id,
4450                            &TranscriptItem::Error(ErrorRecord {
4451                                message: error.clone(),
4452                            }),
4453                        )
4454                        .await?;
4455                        self.emit(RoderEvent::TurnFailed(TurnFailed {
4456                            thread_id: req.thread_id.clone(),
4457                            turn_id: turn_id.clone(),
4458                            error: error.clone(),
4459                            error_kind: None,
4460                            usage: None,
4461                            timestamp: OffsetDateTime::now_utc(),
4462                        }))
4463                        .await;
4464                        self.complete_team_member_turn_with_result(
4465                            &req.thread_id,
4466                            &turn_id,
4467                            TeamMemberStatus::Failed,
4468                            (!assistant_text.trim().is_empty()).then(|| assistant_text.clone()),
4469                            Some(error),
4470                        )
4471                        .await?;
4472                        return Ok(TurnRunOutcome::Stopped);
4473                    }
4474                    InferenceEvent::Usage(_) => {
4475                        self.sync_turn_goal_usage(&req.thread_id, &turn_id).await?;
4476                    }
4477                    InferenceEvent::Completed(metadata) => {
4478                        sampling_output
4479                            .finish(
4480                                self,
4481                                &output_context,
4482                                &assistant_text,
4483                                &phase_messages,
4484                                &reasoning_text,
4485                                &mut transcript,
4486                            )
4487                            .await?;
4488                        provider_stream_retry_attempts = 0;
4489                        turn_finish_reason = metadata
4490                            .stop_reason
4491                            .as_deref()
4492                            .map(finish_reason_from_stop_reason);
4493                        break;
4494                    }
4495                    InferenceEvent::Compaction(progress) => {
4496                        // Persist the boundary as soon as the provider emits a
4497                        // completed compaction item. ChatGPT Codex often aborts
4498                        // the SSE stream right after ("error decoding response
4499                        // body"), so waiting for ProviderMetadata loses the
4500                        // boundary and every subsequent request re-compacts.
4501                        if progress.status == "completed"
4502                            && let Some(item) = progress.item
4503                        {
4504                            let already = stream_compaction_item
4505                                .as_ref()
4506                                .and_then(|existing| {
4507                                    existing.get("id").and_then(serde_json::Value::as_str)
4508                                })
4509                                .zip(item.get("id").and_then(serde_json::Value::as_str))
4510                                .is_some_and(|(a, b)| a == b);
4511                            if !already {
4512                                stream_compaction_item = Some(item.clone());
4513                                let boundary = TranscriptItem::ProviderMetadata(
4514                                    crate::compaction::provider_metadata_from_compaction_item(item),
4515                                );
4516                                self.persist_turn_item(&req.thread_id, &turn_id, &boundary)
4517                                    .await?;
4518                                transcript.push(boundary);
4519                                transcript =
4520                                    crate::compaction::trim_to_last_compaction_boundary(transcript);
4521                                compacted_this_turn = true;
4522                            }
4523                        }
4524                    }
4525                    InferenceEvent::HostedToolCallStarted(_)
4526                    | InferenceEvent::HostedToolCallCompleted(_) => {}
4527                    InferenceEvent::ToolCallStarted(call) => {
4528                        patch_progress.start(&call.id, &call.name)
4529                    }
4530                    InferenceEvent::ToolCallDelta(delta) => {
4531                        if let Some(progress) = patch_progress.delta(
4532                            &req.thread_id,
4533                            &turn_id,
4534                            &delta.id,
4535                            &delta.arguments_delta,
4536                        ) {
4537                            self.emit(RoderEvent::PatchProgress(progress)).await;
4538                        }
4539                    }
4540                    InferenceEvent::ProviderMetadata(mut metadata) => {
4541                        if metadata.get("kind").and_then(serde_json::Value::as_str)
4542                            == Some("reliability_retry_attempt")
4543                        {
4544                            provider_stream_retry_attempts =
4545                                provider_stream_retry_attempts.saturating_add(1);
4546                        }
4547                        provider_requests_continuation |= metadata
4548                            .get("end_turn")
4549                            .and_then(serde_json::Value::as_bool)
4550                            == Some(false);
4551                        if let Some(compaction_item) = stream_compaction_item.as_ref() {
4552                            let _ = crate::compaction::ensure_provider_metadata_compaction(
4553                                &mut metadata,
4554                                compaction_item,
4555                            );
4556                        }
4557                        if metadata.get("output").is_none() {
4558                            let item = TranscriptItem::ProviderMetadata(metadata);
4559                            self.persist_turn_item(&req.thread_id, &turn_id, &item)
4560                                .await?;
4561                            transcript.push(item);
4562                        } else {
4563                            provider_metadata = Some(metadata);
4564                        }
4565                    }
4566                }
4567            }
4568
4569            speed_policy.record_model_output(
4570                !assistant_text.is_empty() || !phase_messages.is_empty(),
4571                tool_calls.len(),
4572            );
4573            if tool_calls.is_empty() {
4574                if provider_requests_continuation || replayed_tool_call {
4575                    if let Some(metadata) = provider_metadata {
4576                        let item = TranscriptItem::ProviderMetadata(metadata);
4577                        self.persist_turn_item(&req.thread_id, &turn_id, &item)
4578                            .await?;
4579                        transcript.push(item);
4580                    }
4581                    continue 'tool_rounds;
4582                }
4583                let steers = self.drain_turn_steers(&turn_id, &mut steering).await;
4584                if !steers.is_empty() {
4585                    if let Some(metadata) = provider_metadata {
4586                        let had_provider_compaction =
4587                            crate::compaction::provider_metadata_has_compaction(&metadata);
4588                        let item = TranscriptItem::ProviderMetadata(metadata);
4589                        self.persist_turn_item(&req.thread_id, &turn_id, &item)
4590                            .await?;
4591                        transcript.push(item);
4592                        if had_provider_compaction {
4593                            // Server-side compaction replaced the prior window;
4594                            // drop pre-boundary items so the next request does
4595                            // not re-send (and re-compact) the full history.
4596                            transcript =
4597                                crate::compaction::trim_to_last_compaction_boundary(transcript);
4598                            compacted_this_turn = true;
4599                        }
4600                    }
4601                    self.append_steers(&req, &turn_id, &mut transcript, steers)
4602                        .await?;
4603                    continue;
4604                }
4605                if !deadline_finalization_requested
4606                    && req.task_ledger_required
4607                    && runtime_profile == RuntimeProfile::Eval
4608                    && task_ledger_completion_reminders < TASK_LEDGER_COMPLETION_REMINDER_LIMIT
4609                    && (!assistant_text.trim().is_empty() || !phase_messages.is_empty())
4610                    && let Some(prompt) = task_ledger_completion_prompt(&transcript)
4611                {
4612                    task_ledger_completion_reminders += 1;
4613                    let item = TranscriptItem::UserMessage(UserMessage::text(prompt));
4614                    self.persist_turn_item(&req.thread_id, &turn_id, &item)
4615                        .await?;
4616                    transcript.push(item);
4617                    continue;
4618                }
4619                if !deadline_finalization_requested
4620                    && let Some(prompt) = verification_gate.blocking_prompt()
4621                {
4622                    speed_policy.record_verification_required();
4623                    self.emit(RoderEvent::VerificationRequired(VerificationRequired {
4624                        thread_id: req.thread_id.clone(),
4625                        turn_id: turn_id.clone(),
4626                        reason: verification_gate.reason(),
4627                        changed_files: verification_gate.changed_files(),
4628                        tool_evidence: verification_gate.tool_evidence.clone(),
4629                        tests_run: verification_gate.tests_run.clone(),
4630                        open_gaps: verification_gate.open_gaps.clone(),
4631                        timestamp: OffsetDateTime::now_utc(),
4632                    }))
4633                    .await;
4634                    let item = TranscriptItem::UserMessage(UserMessage::text(prompt));
4635                    self.persist_turn_item(&req.thread_id, &turn_id, &item)
4636                        .await?;
4637                    transcript.push(item);
4638                    continue;
4639                }
4640                // Loop persistence nudge: in non-interactive/eval runs, when the
4641                // model returns a final message with no tool calls, gently prompt
4642                // it to keep going (bounded by `empty_tool_call_nudges`) before
4643                // ending the turn. Gated to non-interactive/eval so interactive
4644                // chat completions are never delayed, and only fires when the
4645                // model actually produced a final answer to re-examine.
4646                if !deadline_finalization_requested
4647                    && runtime_profile != RuntimeProfile::Interactive
4648                    && cfg.reliability.empty_tool_call_nudges > 0
4649                    && empty_tool_call_nudges_used < cfg.reliability.empty_tool_call_nudges
4650                    && (!assistant_text.trim().is_empty() || !phase_messages.is_empty())
4651                {
4652                    empty_tool_call_nudges_used += 1;
4653                    let item = TranscriptItem::UserMessage(UserMessage::text(
4654                        RELIABILITY_CONTINUATION_PROMPT.to_string(),
4655                    ));
4656                    self.persist_turn_item(&req.thread_id, &turn_id, &item)
4657                        .await?;
4658                    transcript.push(item);
4659                    continue;
4660                }
4661                if deadline_finalization_requested
4662                    && assistant_text.trim().is_empty()
4663                    && phase_messages.is_empty()
4664                {
4665                    assistant_text = format!(
4666                        "Deadline finalization completed without model text. {}",
4667                        turn_partial_result(&transcript)
4668                    );
4669                }
4670                final_assistant_text = assistant_text;
4671                final_provider_metadata = provider_metadata;
4672                exhausted_tool_rounds = false;
4673                break;
4674            }
4675
4676            if let Some(metadata) = provider_metadata {
4677                let had_provider_compaction =
4678                    crate::compaction::provider_metadata_has_compaction(&metadata);
4679                let item = TranscriptItem::ProviderMetadata(metadata);
4680                self.persist_turn_item(&req.thread_id, &turn_id, &item)
4681                    .await?;
4682                transcript.push(item);
4683                if had_provider_compaction {
4684                    // Server-side compaction replaced the prior window; drop
4685                    // pre-boundary items so subsequent tool rounds use the
4686                    // compact window as the next input.
4687                    transcript = crate::compaction::trim_to_last_compaction_boundary(transcript);
4688                    compacted_this_turn = true;
4689                }
4690            }
4691            eager_tools.drain().await?;
4692            for call in &tool_calls {
4693                if eager_tools.scheduled(&call.id) {
4694                    continue;
4695                }
4696                let tool_item = TranscriptItem::ToolCall(ToolCallRecord {
4697                    id: call.id.clone(),
4698                    name: call.name.clone(),
4699                    arguments: call.arguments.clone(),
4700                });
4701                self.persist_turn_item(&req.thread_id, &turn_id, &tool_item)
4702                    .await?;
4703                transcript.push(tool_item);
4704                self.persist_model_profile_segment(
4705                    &req.thread_id,
4706                    &turn_id,
4707                    model_profile.as_ref(),
4708                    &provider,
4709                    &model,
4710                    "tool_call",
4711                )
4712                .await?;
4713            }
4714            if let Some(deadline) = turn_deadline
4715                && deadline_expired(deadline)
4716            {
4717                self.fail_turn_due_to_deadline(&req.thread_id, &turn_id, deadline, &transcript)
4718                    .await?;
4719                return Ok(TurnRunOutcome::Stopped);
4720            }
4721            let mut results = eager_tools.take_results(&tool_calls);
4722            let remaining_calls = tool_calls
4723                .into_iter()
4724                .filter(|call| !eager_tools.scheduled(&call.id))
4725                .collect();
4726            results.extend(
4727                self.route_tool_calls(
4728                    &req.thread_id,
4729                    &turn_id,
4730                    remaining_calls,
4731                    parallel_tool_calls,
4732                    Some(workspace.as_str()),
4733                    turn_deadline,
4734                )
4735                .await?,
4736            );
4737            let reliability_limit = reliability.record_tool_results(
4738                &cfg.reliability,
4739                &results,
4740                runtime_profile == RuntimeProfile::Interactive,
4741            );
4742            for result in results {
4743                verification_gate.record_tool_result(&result);
4744                transcript.push(TranscriptItem::ToolResult(result));
4745                self.persist_model_profile_segment(
4746                    &req.thread_id,
4747                    &turn_id,
4748                    model_profile.as_ref(),
4749                    &provider,
4750                    &model,
4751                    "tool_result",
4752                )
4753                .await?;
4754            }
4755            if let Some(limit) = reliability_limit {
4756                if limit.decision == ReliabilityLimitDecision::RequestContinuation {
4757                    // Loop persistence: rather than ending the turn, reset the
4758                    // consecutive-failure counter, nudge the model to keep going,
4759                    // and continue the round loop. Bounded by the per-turn tool
4760                    // failure ceiling, `max_model_calls_per_turn`, and
4761                    // `MAX_TOOL_ROUNDS_PER_TURN`.
4762                    self.record_reliability_limit_continuation(
4763                        &req.thread_id,
4764                        &turn_id,
4765                        &provider,
4766                        &model,
4767                        limit,
4768                    )
4769                    .await;
4770                    reliability.reset_consecutive_failures();
4771                    let nudge = TranscriptItem::UserMessage(UserMessage::text(
4772                        RELIABILITY_CONTINUATION_PROMPT.to_string(),
4773                    ));
4774                    self.persist_turn_item(&req.thread_id, &turn_id, &nudge)
4775                        .await?;
4776                    transcript.push(nudge);
4777                } else {
4778                    self.fail_turn_due_to_reliability_limit(
4779                        &req.thread_id,
4780                        &turn_id,
4781                        &provider,
4782                        &model,
4783                        limit,
4784                        &transcript,
4785                    )
4786                    .await?;
4787                    return Ok(TurnRunOutcome::Stopped);
4788                }
4789            }
4790            let compaction_generation_before = self.compaction_generation(&req.thread_id);
4791            transcript = self
4792                .compact_transcript_if_needed(
4793                    &req.thread_id,
4794                    &turn_id,
4795                    &provider,
4796                    &model,
4797                    transcript,
4798                    self.compaction_options_for_turn(&req.thread_id, !compacted_this_turn),
4799                )
4800                .await?;
4801            compacted_this_turn |=
4802                self.compaction_generation(&req.thread_id) != compaction_generation_before;
4803        }
4804
4805        if exhausted_tool_rounds {
4806            let message =
4807                format!("tool call limit reached after {MAX_TOOL_ROUNDS_PER_TURN} rounds");
4808            self.persist_turn_item(
4809                &req.thread_id,
4810                &turn_id,
4811                &TranscriptItem::Error(ErrorRecord {
4812                    message: message.clone(),
4813                }),
4814            )
4815            .await?;
4816            self.emit(RoderEvent::TurnFailed(TurnFailed {
4817                thread_id: req.thread_id.clone(),
4818                turn_id: turn_id.clone(),
4819                error: message.clone(),
4820                error_kind: None,
4821                usage: None,
4822                timestamp: OffsetDateTime::now_utc(),
4823            }))
4824            .await;
4825            self.complete_team_member_turn_with_result(
4826                &req.thread_id,
4827                &turn_id,
4828                TeamMemberStatus::Failed,
4829                None,
4830                Some(message),
4831            )
4832            .await?;
4833            return Ok(TurnRunOutcome::Stopped);
4834        }
4835
4836        if let Some(metadata) = final_provider_metadata {
4837            self.persist_turn_item(
4838                &req.thread_id,
4839                &turn_id,
4840                &TranscriptItem::ProviderMetadata(metadata),
4841            )
4842            .await?;
4843        }
4844
4845        let (cleanup, ownership) = self.await_provider_turn_cleanup(&turn_id).await;
4846        self.record_turn_lifecycle_with_ownership(
4847            req.thread_id.clone(),
4848            turn_id.clone(),
4849            TurnLifecycleState::Completed,
4850            cleanup,
4851            None,
4852            ownership,
4853        )
4854        .await;
4855        self.emit(RoderEvent::TurnCompleted(TurnCompleted {
4856            thread_id: req.thread_id.clone(),
4857            turn_id: turn_id.clone(),
4858            usage: None,
4859            finish_reason: turn_finish_reason,
4860            timestamp: OffsetDateTime::now_utc(),
4861        }))
4862        .await;
4863        self.complete_team_member_turn_with_result(
4864            &req.thread_id,
4865            &turn_id,
4866            TeamMemberStatus::Completed,
4867            (!final_assistant_text.is_empty()).then_some(final_assistant_text),
4868            None,
4869        )
4870        .await?;
4871        Ok(TurnRunOutcome::Completed)
4872    }
4873
4874    async fn route_tool_calls(
4875        self: &Arc<Self>,
4876        thread_id: &ThreadId,
4877        turn_id: &TurnId,
4878        calls: Vec<ToolCallCompleted>,
4879        parallel: bool,
4880        workspace: Option<&str>,
4881        deadline: Option<OffsetDateTime>,
4882    ) -> anyhow::Result<Vec<ToolResultRecord>> {
4883        // Enforce the agent_swarm exclusivity rule (roadmap 104, Task 2):
4884        // `agent_swarm` must be the only tool call in a model response. A mixed
4885        // or multi-swarm batch is denied wholesale with actionable retry text so
4886        // the model re-issues `agent_swarm` by itself, and every tool_call_id
4887        // still gets a response (keeping the chat-completions transcript valid).
4888        if let Some(violation) = roder_api::subagents::agent_swarm_batch_violation(
4889            calls.iter().map(|call| call.name.as_str()),
4890        ) {
4891            let message = violation.deny_message();
4892            return Ok(calls
4893                .into_iter()
4894                .map(|call| ToolResultRecord {
4895                    id: call.id,
4896                    name: Some(call.name),
4897                    result: message.clone(),
4898                    display_payload: None,
4899                    is_error: true,
4900                })
4901                .collect());
4902        }
4903        // Emit swarm lifecycle events on the bus (roadmap 104, Task 1) so any
4904        // app-server/SDK/TUI client can observe a swarm as a whole; per-child
4905        // progress flows through the existing Subagent* trace events.
4906        let swarm_call = (calls.len() == 1
4907            && calls[0].name == roder_api::subagents::AGENT_SWARM_TOOL_NAME)
4908            .then(|| (calls[0].id.clone(), calls[0].arguments.clone()));
4909        if let Some((tool_id, args)) = &swarm_call {
4910            self.emit(RoderEvent::AgentSwarmStarted(
4911                roder_api::subagents::AgentSwarmStarted {
4912                    thread_id: thread_id.clone(),
4913                    turn_id: turn_id.clone(),
4914                    tool_id: tool_id.clone(),
4915                    child_count: agent_swarm_child_count(args),
4916                    timestamp: OffsetDateTime::now_utc(),
4917                },
4918            ))
4919            .await;
4920        }
4921
4922        let force_sequential = calls.iter().any(|call| {
4923            crate::agent_control_tools::is_agent_control_tool(&call.name)
4924                || crate::thread_events::is_event_tool(&call.name)
4925        });
4926        let results = if parallel && !force_sequential {
4927            try_join_all(calls.into_iter().map(|call| {
4928                self.route_tool_call(thread_id, turn_id, call, workspace, deadline, None)
4929            }))
4930            .await
4931        } else {
4932            let mut results = Vec::with_capacity(calls.len());
4933            for call in calls {
4934                results.push(
4935                    self.route_tool_call(thread_id, turn_id, call, workspace, deadline, None)
4936                        .await?,
4937                );
4938            }
4939            Ok(results)
4940        }?;
4941
4942        if let Some((tool_id, _)) = &swarm_call
4943            && let Some(result) = results.iter().find(|result| &result.id == tool_id)
4944            && let Some((completed, failed, aborted)) = parse_swarm_counts(&result.result)
4945        {
4946            self.emit(RoderEvent::AgentSwarmCompleted(
4947                roder_api::subagents::AgentSwarmCompleted {
4948                    thread_id: thread_id.clone(),
4949                    turn_id: turn_id.clone(),
4950                    tool_id: tool_id.clone(),
4951                    completed,
4952                    failed,
4953                    aborted,
4954                    timestamp: OffsetDateTime::now_utc(),
4955                },
4956            ))
4957            .await;
4958        }
4959
4960        Ok(results)
4961    }
4962
4963    /// On provider context/prompt-length failures, force-compact (and if needed
4964    /// strip the last bulky item) then retry the current tool round instead of
4965    /// failing the turn. Bounded so a permanently oversized suffix still stops.
4966    async fn try_recover_from_context_limit(
4967        &self,
4968        thread_id: &ThreadId,
4969        turn_id: &TurnId,
4970        provider: &str,
4971        model: &str,
4972        error: &str,
4973        typed_overflow: bool,
4974        transcript: &mut Vec<TranscriptItem>,
4975        compacted_this_turn: &mut bool,
4976        recovery_attempts: &mut u32,
4977    ) -> anyhow::Result<bool> {
4978        const MAX_CONTEXT_LIMIT_RECOVERY_ATTEMPTS: u32 = 2;
4979        if !typed_overflow && !crate::compaction::is_context_limit_failure_message(error) {
4980            return Ok(false);
4981        }
4982        if *recovery_attempts >= MAX_CONTEXT_LIMIT_RECOVERY_ATTEMPTS {
4983            return Ok(false);
4984        }
4985        *recovery_attempts = recovery_attempts.saturating_add(1);
4986
4987        let error_item = TranscriptItem::Error(ErrorRecord {
4988            message: error.to_string(),
4989        });
4990        self.persist_turn_item(thread_id, turn_id, &error_item)
4991            .await?;
4992        transcript.push(error_item);
4993
4994        // After the first failed recovery, drop the item ahead of the latest
4995        // user/error so a second compact can succeed when the suffix itself is
4996        // still oversized (e.g. a huge tool result right before the prompt).
4997        if *recovery_attempts > 1 {
4998            *transcript =
4999                crate::compaction::strip_last_message_before_error(std::mem::take(transcript));
5000        }
5001
5002        let force_options = crate::compaction::CompactionOptions {
5003            allow_repeat: true,
5004            force: true,
5005            hysteresis_baseline: None,
5006            preserve_hint: Some(
5007                "Recover from a provider context/prompt-length limit; preserve the active user goal and recent tool outcomes."
5008                    .to_string(),
5009            ),
5010        };
5011        let before_len = transcript.len();
5012        let tool_images = self.tool_images(provider, model);
5013        let before_tokens = crate::compaction::estimate_prompt_tokens(transcript, tool_images);
5014        *transcript = self
5015            .compact_transcript_if_needed(
5016                thread_id,
5017                turn_id,
5018                provider,
5019                model,
5020                std::mem::take(transcript),
5021                force_options,
5022            )
5023            .await?;
5024        let after_tokens = crate::compaction::estimate_prompt_tokens(transcript, tool_images);
5025        *compacted_this_turn = *compacted_this_turn
5026            || transcript
5027                .iter()
5028                .any(crate::compaction::is_compaction_boundary);
5029
5030        self.emit(RoderEvent::ReliabilityRetryRecorded(
5031            ReliabilityRetryRecorded {
5032                context: ReliabilityContext {
5033                    thread_id: thread_id.clone(),
5034                    turn_id: turn_id.clone(),
5035                    provider: Some(provider.to_string()),
5036                    model: Some(model.to_string()),
5037                    ..ReliabilityContext::default()
5038                },
5039                error_class: ReliabilityErrorClass::ProviderError,
5040                decision: ReliabilityRetryDecision::Retry,
5041                attempt: *recovery_attempts,
5042                max_attempts: MAX_CONTEXT_LIMIT_RECOVERY_ATTEMPTS,
5043                delay_ms: Some(0),
5044                details: ReliabilityDetails::redacted(format!(
5045                    "context_limit_recovery: compact {before_tokens}->{after_tokens} tokens (items {before_len}->{}); {error}",
5046                    transcript.len()
5047                )),
5048                timestamp: OffsetDateTime::now_utc(),
5049            },
5050        ))
5051        .await;
5052
5053        // If force-compact did not shrink anything and we still have budget,
5054        // strip once more so the next round is not identical.
5055        if after_tokens >= before_tokens && *recovery_attempts < MAX_CONTEXT_LIMIT_RECOVERY_ATTEMPTS
5056        {
5057            *transcript =
5058                crate::compaction::strip_last_message_before_error(std::mem::take(transcript));
5059        }
5060        Ok(true)
5061    }
5062
5063    async fn fail_turn_with_error(
5064        &self,
5065        thread_id: &ThreadId,
5066        turn_id: &TurnId,
5067        message: String,
5068    ) -> anyhow::Result<()> {
5069        self.persist_turn_item(
5070            thread_id,
5071            turn_id,
5072            &TranscriptItem::Error(ErrorRecord {
5073                message: message.clone(),
5074            }),
5075        )
5076        .await?;
5077        self.emit(RoderEvent::TurnFailed(TurnFailed {
5078            thread_id: thread_id.clone(),
5079            turn_id: turn_id.clone(),
5080            error: message.clone(),
5081            error_kind: None,
5082            usage: None,
5083            timestamp: OffsetDateTime::now_utc(),
5084        }))
5085        .await;
5086        self.complete_team_member_turn_with_result(
5087            thread_id,
5088            turn_id,
5089            TeamMemberStatus::Failed,
5090            None,
5091            Some(message),
5092        )
5093        .await?;
5094        Ok(())
5095    }
5096
5097    async fn fail_turn_due_to_deadline(
5098        &self,
5099        thread_id: &ThreadId,
5100        turn_id: &TurnId,
5101        deadline: OffsetDateTime,
5102        transcript: &[TranscriptItem],
5103    ) -> anyhow::Result<()> {
5104        let partial_result = turn_partial_result(transcript);
5105        self.emit(RoderEvent::TurnPartialResult(TurnPartialResult {
5106            thread_id: thread_id.clone(),
5107            turn_id: turn_id.clone(),
5108            summary: partial_result.clone(),
5109            timestamp: OffsetDateTime::now_utc(),
5110        }))
5111        .await;
5112        self.emit(RoderEvent::TurnDeadlineExceeded(TurnDeadlineExceeded {
5113            thread_id: thread_id.clone(),
5114            turn_id: turn_id.clone(),
5115            deadline,
5116            partial_result: partial_result.clone(),
5117            timestamp: OffsetDateTime::now_utc(),
5118        }))
5119        .await;
5120        let message = "turn deadline expired".to_string();
5121        self.persist_turn_item(
5122            thread_id,
5123            turn_id,
5124            &TranscriptItem::Error(ErrorRecord {
5125                message: format!("{message}: {partial_result}"),
5126            }),
5127        )
5128        .await?;
5129        self.emit(RoderEvent::TurnFailed(TurnFailed {
5130            thread_id: thread_id.clone(),
5131            turn_id: turn_id.clone(),
5132            error: message.clone(),
5133            error_kind: Some("deadline_timeout".to_string()),
5134            usage: None,
5135            timestamp: OffsetDateTime::now_utc(),
5136        }))
5137        .await;
5138        self.complete_team_member_turn_with_result(
5139            thread_id,
5140            turn_id,
5141            TeamMemberStatus::Failed,
5142            None,
5143            Some(format!("{message}: {partial_result}")),
5144        )
5145        .await?;
5146        Ok(())
5147    }
5148
5149    async fn start_deadline_finalization(
5150        &self,
5151        thread_id: &ThreadId,
5152        turn_id: &TurnId,
5153        transcript: &mut Vec<TranscriptItem>,
5154        remaining_seconds: u64,
5155    ) -> anyhow::Result<()> {
5156        let item = TranscriptItem::UserMessage(crate::deadline_policy::finalization_message(
5157            remaining_seconds,
5158        ));
5159        self.persist_turn_item(thread_id, turn_id, &item).await?;
5160        transcript.push(item);
5161        self.emit(RoderEvent::TurnPartialResult(TurnPartialResult {
5162            thread_id: thread_id.clone(),
5163            turn_id: turn_id.clone(),
5164            summary: turn_partial_result(transcript),
5165            timestamp: OffsetDateTime::now_utc(),
5166        }))
5167        .await;
5168        Ok(())
5169    }
5170
5171    /// Emits the reliability-limit event for a continuation (loop persistence)
5172    /// without failing the turn. The caller resets the failure counter and
5173    /// injects a nudge so the round loop keeps going.
5174    async fn record_reliability_limit_continuation(
5175        &self,
5176        thread_id: &ThreadId,
5177        turn_id: &TurnId,
5178        provider: &str,
5179        model: &str,
5180        limit: ReliabilityLimitHit,
5181    ) {
5182        self.emit(RoderEvent::ReliabilityLimitRecorded(
5183            ReliabilityLimitRecorded {
5184                context: ReliabilityContext {
5185                    thread_id: thread_id.clone(),
5186                    turn_id: turn_id.clone(),
5187                    tool_id: None,
5188                    tool_name: None,
5189                    provider: Some(provider.to_string()),
5190                    model: Some(model.to_string()),
5191                },
5192                error_class: limit.error_class,
5193                limit_kind: limit.limit_kind,
5194                decision: limit.decision,
5195                current: limit.current,
5196                limit: limit.limit,
5197                details: ReliabilityDetails::redacted(&limit.message),
5198                timestamp: OffsetDateTime::now_utc(),
5199            },
5200        ))
5201        .await;
5202    }
5203
5204    async fn fail_turn_due_to_reliability_limit(
5205        &self,
5206        thread_id: &ThreadId,
5207        turn_id: &TurnId,
5208        provider: &str,
5209        model: &str,
5210        limit: ReliabilityLimitHit,
5211        transcript: &[TranscriptItem],
5212    ) -> anyhow::Result<()> {
5213        self.emit(RoderEvent::ReliabilityLimitRecorded(
5214            ReliabilityLimitRecorded {
5215                context: ReliabilityContext {
5216                    thread_id: thread_id.clone(),
5217                    turn_id: turn_id.clone(),
5218                    tool_id: None,
5219                    tool_name: None,
5220                    provider: Some(provider.to_string()),
5221                    model: Some(model.to_string()),
5222                },
5223                error_class: limit.error_class,
5224                limit_kind: limit.limit_kind,
5225                decision: limit.decision,
5226                current: limit.current,
5227                limit: limit.limit,
5228                details: ReliabilityDetails::redacted(&limit.message),
5229                timestamp: OffsetDateTime::now_utc(),
5230            },
5231        ))
5232        .await;
5233        let partial_result = turn_partial_result(transcript);
5234        self.emit(RoderEvent::TurnPartialResult(TurnPartialResult {
5235            thread_id: thread_id.clone(),
5236            turn_id: turn_id.clone(),
5237            summary: partial_result.clone(),
5238            timestamp: OffsetDateTime::now_utc(),
5239        }))
5240        .await;
5241        let message = format!("reliability limit reached: {}", limit.message);
5242        self.persist_turn_item(
5243            thread_id,
5244            turn_id,
5245            &TranscriptItem::Error(ErrorRecord {
5246                message: format!("{message}: {partial_result}"),
5247            }),
5248        )
5249        .await?;
5250        self.emit(RoderEvent::TurnFailed(TurnFailed {
5251            thread_id: thread_id.clone(),
5252            turn_id: turn_id.clone(),
5253            error: message.clone(),
5254            error_kind: Some("reliability_limit".to_string()),
5255            usage: None,
5256            timestamp: OffsetDateTime::now_utc(),
5257        }))
5258        .await;
5259        self.complete_team_member_turn_with_result(
5260            thread_id,
5261            turn_id,
5262            TeamMemberStatus::Failed,
5263            None,
5264            Some(format!("{message}: {partial_result}")),
5265        )
5266        .await?;
5267        Ok(())
5268    }
5269
5270    async fn append_steers(
5271        &self,
5272        req: &StartTurnRequest,
5273        turn_id: &TurnId,
5274        transcript: &mut Vec<TranscriptItem>,
5275        steers: Vec<QueuedTurnSteer>,
5276    ) -> anyhow::Result<()> {
5277        for queued in steers {
5278            let mut steer = queued.message;
5279            steer.text = steer.text.trim().to_string();
5280            if steer.text.is_empty() && steer.images.is_empty() {
5281                continue;
5282            }
5283            let item = TranscriptItem::UserMessage(steer);
5284            self.persist_turn_item(&req.thread_id, turn_id, &item)
5285                .await?;
5286            transcript.push(item);
5287            if let Some(ack) = queued.mailbox_ack {
5288                self.teams
5289                    .mark_mailbox_messages_delivered(&ack.team_id, turn_id, &ack.message_ids)
5290                    .await?;
5291            }
5292        }
5293        Ok(())
5294    }
5295
5296    async fn persist_model_profile_segment(
5297        &self,
5298        thread_id: &ThreadId,
5299        turn_id: &TurnId,
5300        profile: Option<&ModelHarnessProfile>,
5301        provider: &str,
5302        model: &str,
5303        segment: &str,
5304    ) -> anyhow::Result<()> {
5305        let item = TranscriptItem::ProviderMetadata(model_profile_segment_metadata(
5306            profile, provider, model, segment,
5307        ));
5308        self.persist_turn_item(thread_id, turn_id, &item).await
5309    }
5310
5311    fn task_ledger_tool_specs(
5312        &self,
5313        profile: Option<&ModelHarnessProfile>,
5314    ) -> Vec<roder_api::tools::ToolSpec> {
5315        self.tool_registry
5316            .get(TASK_LEDGER_TOOL_NAME)
5317            .map(|tool| {
5318                tool.spec()
5319                    .normalized_for_model_profile(schema_policy_for_model(profile))
5320            })
5321            .into_iter()
5322            .collect()
5323    }
5324
5325    pub(crate) fn engine_for(&self, provider: &str) -> anyhow::Result<Arc<dyn InferenceEngine>> {
5326        self.registry
5327            .inference_engine(provider)
5328            .or_else(|| {
5329                self.registry
5330                    .default_inference_engine()
5331                    .filter(|engine| provider.is_empty() || engine.id() == provider)
5332            })
5333            .ok_or_else(|| anyhow::anyhow!("inference provider {provider:?} is not registered"))
5334    }
5335
5336    pub async fn emit(&self, mut event: RoderEvent) -> EventEnvelope {
5337        self.account_usage_event(&mut event).await;
5338        self.dispatch_local_hook_lifecycle(&event).await;
5339        if let Some(record) = Self::lifecycle_record_for_event(&event) {
5340            let _ = self.persist_turn_lifecycle_record(&record).await;
5341        }
5342        let envelope = self.bus.emit(event);
5343        if let (Some(store), Some(thread_id)) = (&self.thread_store, envelope.thread_id.as_ref())
5344            && should_persist_thread_event(thread_id)
5345        {
5346            let _ = store.append_event(thread_id, &envelope).await;
5347        }
5348        self.dispatch_event_sinks(&envelope).await;
5349        envelope
5350    }
5351
5352    async fn dispatch_local_hook_lifecycle(&self, event: &RoderEvent) {
5353        // SessionStart belongs to the session, not the turn: firing it from
5354        // TurnStarted re-ran setup hooks on every user message. ThreadCreated
5355        // and ThreadLoaded carry no turn id, so they borrow a synthetic one the
5356        // way the SessionEnd path does.
5357        let session_turn_id = SESSION_HOOK_TURN_ID.to_string();
5358        let (thread_id, turn_id, name, matcher, input) = match event {
5359            RoderEvent::ThreadCreated(event) => (
5360                &event.thread_id,
5361                &session_turn_id,
5362                "SessionStart",
5363                Some("startup"),
5364                serde_json::json!({"source":"startup"}),
5365            ),
5366            RoderEvent::ThreadLoaded(event) => (
5367                &event.thread_id,
5368                &session_turn_id,
5369                "SessionStart",
5370                Some("resume"),
5371                serde_json::json!({"source":"resume"}),
5372            ),
5373            RoderEvent::TurnCompleted(event) => (
5374                &event.thread_id,
5375                &event.turn_id,
5376                "Stop",
5377                None,
5378                serde_json::json!({"finishReason":event.finish_reason}),
5379            ),
5380            RoderEvent::TurnFailed(event) => (
5381                &event.thread_id,
5382                &event.turn_id,
5383                "Stop",
5384                None,
5385                serde_json::json!({"error":event.error}),
5386            ),
5387            RoderEvent::ContextCompactionStarted(event) => (
5388                &event.thread_id,
5389                &event.turn_id,
5390                "PreCompact",
5391                None,
5392                serde_json::json!({"originalItemCount":event.original_item_count,"originalEstimatedTokens":event.original_estimated_tokens}),
5393            ),
5394            RoderEvent::ContextCompactionRecorded(event) => (
5395                &event.thread_id,
5396                &event.turn_id,
5397                "PostCompact",
5398                None,
5399                serde_json::json!({"compactedItemCount":event.compacted_item_count,"compactedEstimatedTokens":event.compacted_estimated_tokens}),
5400            ),
5401            RoderEvent::SubagentStarted(event) => (
5402                &event.parent_thread_id,
5403                &event.parent_turn_id,
5404                "SubagentStart",
5405                Some(event.agent_type.as_str()),
5406                serde_json::json!({"subagentThreadId":event.thread_id,"agentType":event.agent_type,"description":event.description}),
5407            ),
5408            RoderEvent::SubagentCompleted(event) => (
5409                &event.parent_thread_id,
5410                &event.parent_turn_id,
5411                "SubagentStop",
5412                Some(event.agent_type.as_str()),
5413                serde_json::json!({"subagentThreadId":event.thread_id,"agentType":event.agent_type,"exitReason":event.exit_reason}),
5414            ),
5415            RoderEvent::SubagentFailed(event) => (
5416                &event.parent_thread_id,
5417                &event.parent_turn_id,
5418                "SubagentStop",
5419                Some(event.agent_type.as_str()),
5420                serde_json::json!({"subagentThreadId":event.thread_id,"agentType":event.agent_type,"error":event.error}),
5421            ),
5422            _ => return,
5423        };
5424        // One SessionStart per session, whichever event opened it.
5425        if name == "SessionStart" && !self.claim_session_hook_start(thread_id).await {
5426            return;
5427        }
5428        let workspace = self.config.read().await.workspace.clone();
5429        crate::hooks::run_lifecycle(
5430            self,
5431            thread_id,
5432            turn_id,
5433            workspace.as_deref(),
5434            name,
5435            matcher,
5436            input,
5437        )
5438        .await;
5439    }
5440
5441    /// Claim the single `SessionStart` dispatch for a thread, returning whether
5442    /// this caller won it.
5443    pub(crate) async fn claim_session_hook_start(&self, thread_id: &ThreadId) -> bool {
5444        self.session_hook_started
5445            .write()
5446            .await
5447            .insert(thread_id.clone())
5448    }
5449
5450    /// Records a projected thread item event and persists it through the configured thread store.
5451    ///
5452    /// App-server protocol notification bridges currently call this after translating runtime
5453    /// events into item events. Headless runtime consumers that subscribe to `RoderEvent` directly
5454    /// must make the same call if they need the derived item event stream persisted.
5455    pub async fn record_thread_item_event_kind(
5456        &self,
5457        thread_id: &ThreadId,
5458        turn_id: &TurnId,
5459        timestamp: OffsetDateTime,
5460        kind: ThreadItemEventKind,
5461    ) -> anyhow::Result<ThreadItemEvent> {
5462        let seq = self.next_thread_item_event_seq(thread_id).await?;
5463        let item_event = ThreadItemEvent {
5464            seq,
5465            event_id: uuid::Uuid::new_v4().to_string(),
5466            thread_id: thread_id.clone(),
5467            turn_id: turn_id.clone(),
5468            timestamp,
5469            event: kind,
5470        };
5471        if let Some(store) = &self.thread_store {
5472            store.append_item_event(thread_id, &item_event).await?;
5473        }
5474        self.remember_thread_item_event(&item_event).await?;
5475        Ok(item_event)
5476    }
5477
5478    async fn next_thread_item_event_seq(&self, thread_id: &ThreadId) -> anyhow::Result<u64> {
5479        self.ensure_thread_item_cache(thread_id).await?;
5480        Ok(self
5481            .thread_item_cache
5482            .lock()
5483            .await
5484            .next_item_event_seq(thread_id))
5485    }
5486
5487    pub async fn thread_item_exists(
5488        &self,
5489        thread_id: &ThreadId,
5490        turn_id: &TurnId,
5491        item_id: &str,
5492    ) -> anyhow::Result<bool> {
5493        self.ensure_thread_item_cache(thread_id).await?;
5494        Ok(self
5495            .thread_item_cache
5496            .lock()
5497            .await
5498            .thread_item_exists(thread_id, turn_id, item_id))
5499    }
5500
5501    pub async fn current_reasoning_item_id(
5502        &self,
5503        thread_id: &ThreadId,
5504        turn_id: &TurnId,
5505    ) -> anyhow::Result<Option<String>> {
5506        self.ensure_thread_item_cache(thread_id).await?;
5507        Ok(self
5508            .thread_item_cache
5509            .lock()
5510            .await
5511            .current_reasoning_item_id(thread_id, turn_id))
5512    }
5513
5514    async fn remember_thread_item_event(&self, item_event: &ThreadItemEvent) -> anyhow::Result<()> {
5515        self.ensure_thread_item_cache(&item_event.thread_id).await?;
5516        self.thread_item_cache
5517            .lock()
5518            .await
5519            .remember_item_event(item_event);
5520        Ok(())
5521    }
5522
5523    pub async fn latest_transcript_item_index(
5524        &self,
5525        thread_id: &ThreadId,
5526        turn_id: &TurnId,
5527    ) -> anyhow::Result<Option<usize>> {
5528        self.ensure_thread_item_cache(thread_id).await?;
5529        Ok(self
5530            .thread_item_cache
5531            .lock()
5532            .await
5533            .latest_transcript_item_index(thread_id, turn_id))
5534    }
5535
5536    async fn next_transcript_item_index(
5537        &self,
5538        thread_id: &ThreadId,
5539        turn_id: &TurnId,
5540    ) -> anyhow::Result<usize> {
5541        self.ensure_thread_item_cache(thread_id).await?;
5542        Ok(self
5543            .thread_item_cache
5544            .lock()
5545            .await
5546            .next_transcript_item_index(thread_id, turn_id))
5547    }
5548
5549    async fn remember_transcript_item_index(
5550        &self,
5551        thread_id: &ThreadId,
5552        turn_id: &TurnId,
5553        item_index: usize,
5554    ) -> anyhow::Result<()> {
5555        self.ensure_thread_item_cache(thread_id).await?;
5556        self.thread_item_cache
5557            .lock()
5558            .await
5559            .remember_transcript_item_index(thread_id, turn_id, item_index);
5560        Ok(())
5561    }
5562
5563    async fn ensure_thread_item_cache(&self, thread_id: &ThreadId) -> anyhow::Result<()> {
5564        if self
5565            .thread_item_cache
5566            .lock()
5567            .await
5568            .contains_thread(thread_id)
5569        {
5570            return Ok(());
5571        }
5572
5573        let snapshot = if let Some(store) = &self.thread_store {
5574            store.load_thread(thread_id).await?
5575        } else {
5576            None
5577        };
5578        self.thread_item_cache.lock().await.ensure_thread(
5579            thread_id,
5580            ThreadItemCacheEntry::from_snapshot(snapshot.as_ref()),
5581        );
5582        Ok(())
5583    }
5584
5585    pub(crate) async fn persist_turn_item(
5586        &self,
5587        thread_id: &ThreadId,
5588        turn_id: &TurnId,
5589        item: &TranscriptItem,
5590    ) -> anyhow::Result<()> {
5591        let item_index = self.next_transcript_item_index(thread_id, turn_id).await?;
5592        let timestamp = OffsetDateTime::now_utc();
5593        self.emit(RoderEvent::TranscriptItemAppended(TranscriptItemAppended {
5594            thread_id: thread_id.clone(),
5595            turn_id: turn_id.clone(),
5596            item_type: match item {
5597                TranscriptItem::UserMessage(_) => "user_message",
5598                TranscriptItem::AssistantMessage(_) => "assistant_message",
5599                TranscriptItem::ReasoningSummary(_) => "reasoning_summary",
5600                TranscriptItem::ToolCall(_) => "tool_call",
5601                TranscriptItem::ToolResult(_) => "tool_result",
5602                TranscriptItem::FileChange(_) => "file_change",
5603                TranscriptItem::ContextCompaction(_) => "context_compaction",
5604                TranscriptItem::Error(_) => "error",
5605                TranscriptItem::ProviderMetadata(_) => "provider_metadata",
5606            }
5607            .to_string(),
5608            item_index: Some(item_index),
5609            item: Some(item.clone()),
5610            timestamp,
5611        }))
5612        .await;
5613        self.remember_transcript_item_index(thread_id, turn_id, item_index)
5614            .await?;
5615        Ok(())
5616    }
5617}
5618
5619fn transcript_has_images(transcript: &[TranscriptItem]) -> bool {
5620    transcript.iter().any(|item| {
5621        matches!(
5622            item,
5623            TranscriptItem::UserMessage(message) if !message.images.is_empty()
5624        )
5625    })
5626}
5627
5628fn transcript_has_task_ledger(transcript: &[TranscriptItem]) -> bool {
5629    transcript.iter().any(|item| {
5630        matches!(
5631            item,
5632            TranscriptItem::ToolResult(result)
5633                if result.name.as_deref() == Some(TASK_LEDGER_TOOL_NAME) && !result.is_error
5634        )
5635    })
5636}
5637
5638fn task_ledger_completion_prompt(transcript: &[TranscriptItem]) -> Option<String> {
5639    let latest = transcript.iter().rev().find_map(|item| match item {
5640        TranscriptItem::ToolResult(result)
5641            if result.name.as_deref() == Some(TASK_LEDGER_TOOL_NAME) && !result.is_error =>
5642        {
5643            Some(result.result.as_str())
5644        }
5645        _ => None,
5646    })?;
5647    if !task_ledger_has_open_items(latest) {
5648        return None;
5649    }
5650
5651    let mut ledger = latest.chars().take(1500).collect::<String>();
5652    if latest.chars().nth(1500).is_some() {
5653        ledger.push_str("...");
5654    }
5655    Some(format!(
5656        "Task Ledger Completion Required: the latest task ledger still has pending or in-progress items. Do not provide a final answer yet. Use tools to complete the remaining scoreable work, create or update any required output files, then call `{TASK_LEDGER_TOOL_NAME}` with every task completed and evidence before finalizing.\n\nLatest ledger:\n{ledger}"
5657    ))
5658}
5659
5660fn task_ledger_deadline_completion_prompt(
5661    remaining_seconds: u64,
5662    reserve_seconds: u64,
5663    completion_prompt: &str,
5664) -> String {
5665    format!(
5666        "Eval deadline scoreable completion: {remaining_seconds} seconds remain in the {reserve_seconds}-second finalization reserve. Do not browse, search, or start slow work. Use the available tools now to create or update the required scoreable output files, run only a quick local check if needed, then update the task ledger to completed before finalizing.\n\n{completion_prompt}"
5667    )
5668}
5669
5670fn task_ledger_scoreable_checkpoint_prompt(
5671    remaining_seconds: u64,
5672    completion_prompt: &str,
5673) -> String {
5674    format!(
5675        "Scoreable Output Checkpoint: {remaining_seconds} seconds remain before the eval deadline. Before any further research, browsing, or long commands, use tools now to ensure the required output file(s) exist with the best evidence-backed answer, even if provisional. If a scoreable file already exists, read it and preserve that candidate unless you have stronger task-specific evidence for a replacement. Do not overwrite a plausible dated, historical, or local-evidence candidate with a current live-page, partial-coverage, or weaker guess merely to refresh the checkpoint. You may continue refining afterward, but do not apologize or finalize until the scoreable file exists and the task ledger is updated.\n\n{completion_prompt}"
5676    )
5677}
5678
5679fn task_ledger_has_open_items(ledger: &str) -> bool {
5680    ledger.lines().any(|line| {
5681        let line = line.trim_start();
5682        line.starts_with("- pending:") || line.starts_with("- in_progress:")
5683    })
5684}
5685
5686fn turn_deadline_for_config(cfg: &RuntimeConfig) -> Option<OffsetDateTime> {
5687    cfg.turn_deadline_seconds
5688        .filter(|seconds| *seconds > 0)
5689        .map(|seconds| OffsetDateTime::now_utc() + Duration::seconds(seconds as i64))
5690}
5691
5692fn deadline_expired(deadline: OffsetDateTime) -> bool {
5693    OffsetDateTime::now_utc() >= deadline
5694}
5695
5696pub(crate) fn deadline_remaining_seconds(deadline: Option<OffsetDateTime>) -> Option<u64> {
5697    let deadline = deadline?;
5698    if deadline <= OffsetDateTime::now_utc() {
5699        return Some(0);
5700    }
5701    Some(
5702        (deadline - OffsetDateTime::now_utc())
5703            .unsigned_abs()
5704            .as_secs()
5705            .max(1),
5706    )
5707}
5708
5709fn deadline_instant(deadline: OffsetDateTime) -> tokio::time::Instant {
5710    let now = OffsetDateTime::now_utc();
5711    if deadline <= now {
5712        return tokio::time::Instant::now();
5713    }
5714    tokio::time::Instant::now() + (deadline - now).unsigned_abs()
5715}
5716
5717fn inference_timeout_deadline(
5718    deadline: Option<OffsetDateTime>,
5719    runtime_profile: RuntimeProfile,
5720    task_ledger_required: bool,
5721    reserve_seconds: u64,
5722    finalization_requested: bool,
5723    task_ledger_scoreable_checkpoints: u8,
5724    transcript: &[TranscriptItem],
5725) -> Option<(OffsetDateTime, InferenceTimeoutAction)> {
5726    let deadline = deadline?;
5727    if runtime_profile == RuntimeProfile::Eval
5728        && task_ledger_required
5729        && !finalization_requested
5730        && task_ledger_scoreable_checkpoints < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
5731        && TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS > reserve_seconds
5732        && task_ledger_completion_prompt(transcript).is_some()
5733    {
5734        let checkpoint_deadline =
5735            deadline - Duration::seconds(TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS as i64);
5736        if checkpoint_deadline > OffsetDateTime::now_utc() {
5737            return Some((
5738                checkpoint_deadline,
5739                InferenceTimeoutAction::ScoreableCheckpoint,
5740            ));
5741        }
5742    }
5743    if runtime_profile == RuntimeProfile::Eval && !finalization_requested {
5744        return Some((
5745            deadline - Duration::seconds(reserve_seconds as i64),
5746            InferenceTimeoutAction::Finalization,
5747        ));
5748    }
5749    Some((deadline, InferenceTimeoutAction::Finalization))
5750}
5751
5752fn turn_partial_result(transcript: &[TranscriptItem]) -> String {
5753    let tool_results = transcript
5754        .iter()
5755        .filter(|item| matches!(item, TranscriptItem::ToolResult(_)))
5756        .count();
5757    let assistant_messages = transcript
5758        .iter()
5759        .filter(|item| matches!(item, TranscriptItem::AssistantMessage(_)))
5760        .count();
5761    format!(
5762        "partial turn state: {} transcript items, {assistant_messages} assistant messages, {tool_results} tool results",
5763        transcript.len()
5764    )
5765}
5766
5767fn reasoning_for_model(cfg: &RuntimeConfig, model: &str) -> ReasoningConfig {
5768    let level = effective_reasoning_for_model(cfg, model);
5769    match level.as_str() {
5770        "" | REASONING_NONE => ReasoningConfig::default(),
5771        level => ReasoningConfig {
5772            enabled: true,
5773            level: Some(level.to_string()),
5774        },
5775    }
5776}
5777
5778fn server_side_compaction_threshold(cfg: &RuntimeConfig, model: &str) -> Option<u32> {
5779    let entry = lookup_model(model)?;
5780    if !entry.supports_compaction {
5781        return None;
5782    }
5783    cfg.auto_compact_token_limit
5784        .or_else(|| {
5785            model_profile_for_model(cfg, model).and_then(|profile| profile.auto_compact_token_limit)
5786        })
5787        .or(Some(entry.auto_compact_token_limit))
5788        .filter(|threshold| *threshold > 0)
5789}
5790
5791pub(crate) fn tool_search_for_provider_model(
5792    cfg: &RuntimeConfig,
5793    provider: &str,
5794    model: &str,
5795) -> ToolSearchConfig {
5796    let mut resolved = cfg.tool_search.clone();
5797    if let Some(provider_config) = cfg.provider_tool_search.get(provider) {
5798        provider_config.apply_to(&mut resolved);
5799    }
5800    if let Some(model_config) = cfg.model_tool_search.get(model) {
5801        model_config.apply_to(&mut resolved);
5802    }
5803    resolved
5804}
5805
5806fn parallel_tool_calls_for_model(cfg: &RuntimeConfig, model: &str) -> bool {
5807    cfg.model_parallel_tool_calls
5808        .get(model)
5809        .copied()
5810        .or_else(|| {
5811            model_profile_for_model(cfg, model).and_then(|profile| profile.parallel_tool_calls)
5812        })
5813        .unwrap_or(true)
5814}
5815
5816/// Count the children an `agent_swarm` call will launch from its arguments
5817/// (item-based spawns plus resumes). Lenient: returns 0 on malformed input.
5818fn agent_swarm_child_count(arguments: &str) -> usize {
5819    serde_json::from_str::<roder_api::subagents::AgentSwarmRequest>(arguments)
5820        .map(|request| request.items.len() + request.resume_agent_ids.len())
5821        .unwrap_or(0)
5822}
5823
5824/// Parse `completed`/`failed`/`aborted` counts from an `<agent_swarm_result>`
5825/// summary line, e.g. `<summary>completed: 2, failed: 1</summary>`. Omitted
5826/// buckets default to 0. Returns `None` when the text is not a swarm result.
5827fn parse_swarm_counts(text: &str) -> Option<(usize, usize, usize)> {
5828    if !text.contains("<agent_swarm_result>") {
5829        return None;
5830    }
5831    let bucket = |label: &str| -> usize {
5832        let needle = format!("{label}: ");
5833        text.find(&needle)
5834            .map(|start| start + needle.len())
5835            .map(|start| {
5836                text[start..]
5837                    .chars()
5838                    .take_while(|c| c.is_ascii_digit())
5839                    .collect::<String>()
5840            })
5841            .and_then(|digits| digits.parse().ok())
5842            .unwrap_or(0)
5843    };
5844    Some((bucket("completed"), bucket("failed"), bucket("aborted")))
5845}
5846
5847fn effective_reasoning_for_model(cfg: &RuntimeConfig, model: &str) -> String {
5848    let base_reasoning = default_effective_reasoning_for_model(cfg, model);
5849    if cfg.dynamic_workflows.effort_profile == DynamicWorkflowEffortProfile::Ultracode {
5850        return ultracode_reasoning_level_for_model(
5851            model,
5852            &cfg.speed_policy.ultracode_reasoning,
5853            &base_reasoning,
5854        );
5855    }
5856    base_reasoning
5857}
5858
5859fn default_effective_reasoning_for_model(cfg: &RuntimeConfig, model: &str) -> String {
5860    let Some(entry) = lookup_model(model) else {
5861        return cfg
5862            .reasoning
5863            .clone()
5864            .unwrap_or_else(|| REASONING_NONE.to_string());
5865    };
5866    if entry.supported_reasoning.is_empty() {
5867        return REASONING_NONE.to_string();
5868    }
5869    cfg.reasoning
5870        .as_deref()
5871        .filter(|reasoning| {
5872            entry
5873                .supported_reasoning
5874                .iter()
5875                .any(|option| option.effort == *reasoning)
5876        })
5877        .map(str::to_string)
5878        .or_else(|| {
5879            model_profile_for_model(cfg, model)
5880                .and_then(|profile| profile.reasoning.orientation)
5881                .filter(|reasoning| {
5882                    entry
5883                        .supported_reasoning
5884                        .iter()
5885                        .any(|option| option.effort == reasoning)
5886                })
5887        })
5888        .unwrap_or_else(|| entry.default_reasoning.to_string())
5889}
5890
5891fn validate_reasoning_effort(model: &str, effort: &str) -> anyhow::Result<()> {
5892    if effort == REASONING_NONE && !model_supports_reasoning(model, effort) {
5893        return Ok(());
5894    }
5895    let Some(entry) = lookup_model(model) else {
5896        return Ok(());
5897    };
5898    if entry
5899        .supported_reasoning
5900        .iter()
5901        .any(|option| option.effort == effort)
5902    {
5903        Ok(())
5904    } else {
5905        anyhow::bail!("model {model} does not support reasoning effort {effort}")
5906    }
5907}
5908
5909fn validate_runtime_config_reasoning(cfg: &RuntimeConfig) -> anyhow::Result<()> {
5910    let Some(reasoning) = cfg.reasoning.as_deref() else {
5911        return Ok(());
5912    };
5913    let Some(entry) = lookup_model(&cfg.default_model) else {
5914        return Ok(());
5915    };
5916    if entry.provider != PROVIDER_GEMINI {
5917        return Ok(());
5918    }
5919    validate_reasoning_effort(&cfg.default_model, reasoning)
5920}
5921
5922fn validate_runtime_inference_router_config(
5923    registry: &ExtensionRegistry,
5924    cfg: &RuntimeConfig,
5925) -> anyhow::Result<()> {
5926    if !cfg.inference_router.enabled {
5927        return Ok(());
5928    }
5929    let Some(router_id) = cfg.inference_router.router_id.as_deref() else {
5930        anyhow::bail!("inference_router.enabled requires inference_router.router");
5931    };
5932    if registry.inference_router(router_id).is_some() {
5933        return Ok(());
5934    }
5935    let available = registry
5936        .inference_routers
5937        .iter()
5938        .map(|router| router.id())
5939        .collect::<Vec<_>>()
5940        .join(", ");
5941    if available.is_empty() {
5942        anyhow::bail!("inference router {router_id:?} is not registered");
5943    }
5944    anyhow::bail!(
5945        "inference router {router_id:?} is not registered; available routers: {available}"
5946    );
5947}
5948
5949fn model_supports_reasoning(model: &str, effort: &str) -> bool {
5950    lookup_model(model)
5951        .map(|entry| {
5952            entry
5953                .supported_reasoning
5954                .iter()
5955                .any(|option| option.effort == effort)
5956        })
5957        .unwrap_or(false)
5958}
5959
5960fn is_final_answer_phase(phase: Option<&str>) -> bool {
5961    phase.is_none_or(|phase| phase.is_empty() || phase == FINAL_ANSWER_PHASE)
5962}
5963
5964fn edit_tool_for_model<'a>(cfg: &'a RuntimeConfig, model: &'a str) -> Option<&'a str> {
5965    cfg.model_edit_tools
5966        .get(model)
5967        .map(String::as_str)
5968        .or_else(|| {
5969            cfg.model_profiles
5970                .get(model)
5971                .and_then(|profile| profile.edit_tool.as_deref())
5972        })
5973        .or_else(|| lookup_model(model).and_then(|entry| entry.edit_tool))
5974        .or(Some(EDIT_TOOL_EDIT))
5975}
5976
5977fn model_profile_for_model(cfg: &RuntimeConfig, model: &str) -> Option<ModelHarnessProfile> {
5978    cfg.model_profiles
5979        .get(model)
5980        .cloned()
5981        .or_else(|| built_in_model_profile(model))
5982}
5983
5984pub(crate) fn allowlist_permits(allowlist: &[String], tool_name: &str) -> bool {
5985    allowlist.is_empty() || allowlist.iter().any(|allowed| allowed == tool_name)
5986}
5987
5988/// Provider-aware profile resolution for the active turn.
5989///
5990/// Many model ids are shared across providers (for example Cursor proxies
5991/// `claude-opus-4-8`). Resolving the harness profile by id alone picks the
5992/// first catalog entry, which assigns cross-provider ids the wrong family and
5993/// instruction overlay (e.g. a `cursor/claude-opus-4-8` turn would otherwise
5994/// inherit the Anthropic overlay). Prefer the explicit `(provider, model)`
5995/// catalog entry, keeping user-configured profile overrides as the top
5996/// precedence.
5997fn model_profile_for_provider_model(
5998    cfg: &RuntimeConfig,
5999    provider: &str,
6000    model: &str,
6001) -> Option<ModelHarnessProfile> {
6002    cfg.model_profiles
6003        .get(model)
6004        .cloned()
6005        .or_else(|| built_in_model_profile_for_provider(provider, model))
6006}
6007
6008fn schema_policy_for_model(profile: Option<&ModelHarnessProfile>) -> ModelSchemaPolicy {
6009    profile
6010        .map(|profile| profile.schema_policy)
6011        .unwrap_or_default()
6012}
6013
6014fn model_profile_segment_metadata(
6015    profile: Option<&ModelHarnessProfile>,
6016    provider: &str,
6017    model: &str,
6018    segment: &str,
6019) -> serde_json::Value {
6020    serde_json::json!({
6021        "kind": MODEL_PROFILE_TRACE_KIND,
6022        "segment": segment,
6023        "provider": provider,
6024        "model": model,
6025        "profileModel": profile.map(|profile| profile.model.as_str()).unwrap_or(model),
6026        "providerFamily": profile.map(|profile| profile.provider_family),
6027        "editTool": profile.and_then(|profile| profile.edit_tool.as_deref()),
6028        "schemaPolicy": profile.map(|profile| profile.schema_policy),
6029        "instructionOverlay": profile.map(|profile| profile.instruction_overlay),
6030        "parallelToolCalls": profile.and_then(|profile| profile.parallel_tool_calls),
6031        "autoCompactTokenLimit": profile.and_then(|profile| profile.auto_compact_token_limit),
6032    })
6033}
6034
6035fn model_switch_summary(
6036    transcript: &[TranscriptItem],
6037    profile: Option<&ModelHarnessProfile>,
6038    provider: &str,
6039    model: &str,
6040    tools: &[roder_api::tools::ToolSpec],
6041) -> Option<String> {
6042    let previous = latest_model_profile_segment(transcript)?;
6043    let previous_model = previous
6044        .get("model")
6045        .and_then(serde_json::Value::as_str)
6046        .unwrap_or_default();
6047    let previous_provider = previous
6048        .get("provider")
6049        .and_then(serde_json::Value::as_str)
6050        .unwrap_or_default();
6051    if previous_model == model && previous_provider == provider {
6052        return None;
6053    }
6054
6055    let previous_profile = previous
6056        .get("profileModel")
6057        .and_then(serde_json::Value::as_str)
6058        .unwrap_or(previous_model);
6059    let current_profile = profile
6060        .map(|profile| profile.model.as_str())
6061        .unwrap_or(model);
6062    let previous_edit_tool = previous
6063        .get("editTool")
6064        .and_then(serde_json::Value::as_str)
6065        .unwrap_or("none");
6066    let current_edit_tool = profile
6067        .and_then(|profile| profile.edit_tool.as_deref())
6068        .unwrap_or("none");
6069    let tool_names = tools
6070        .iter()
6071        .map(|tool| tool.name.as_str())
6072        .take(12)
6073        .collect::<Vec<_>>()
6074        .join(", ");
6075    Some(format!(
6076        "{MODEL_SWITCH_SUMMARY_PREFIX} previous profile {previous_provider}/{previous_profile} used edit tool {previous_edit_tool}. Current profile {provider}/{current_profile} uses edit tool {current_edit_tool}. Available tools now: {}.",
6077        if tool_names.is_empty() {
6078            "none"
6079        } else {
6080            &tool_names
6081        }
6082    ))
6083}
6084
6085fn latest_model_profile_segment(transcript: &[TranscriptItem]) -> Option<&serde_json::Value> {
6086    transcript.iter().rev().find_map(|item| {
6087        let TranscriptItem::ProviderMetadata(value) = item else {
6088            return None;
6089        };
6090        (value.get("kind").and_then(serde_json::Value::as_str) == Some(MODEL_PROFILE_TRACE_KIND))
6091            .then_some(value)
6092    })
6093}
6094
6095pub fn validate_edit_tool(value: &str) -> anyhow::Result<()> {
6096    match value.trim() {
6097        EDIT_TOOL_PATCH | EDIT_TOOL_EDIT => Ok(()),
6098        _ => anyhow::bail!(
6099            "unsupported edit_tool {value:?}; allowed values: {EDIT_TOOL_PATCH}, {EDIT_TOOL_EDIT}"
6100        ),
6101    }
6102}
6103
6104fn should_persist_thread_event(thread_id: &str) -> bool {
6105    !is_synthetic_event_thread_id(thread_id)
6106}
6107
6108#[cfg(test)]
6109#[path = "runtime/item_identity_tests.rs"]
6110mod item_identity_tests;
6111
6112#[cfg(test)]
6113#[path = "runtime/codex_v2_tests.rs"]
6114mod codex_v2_tests;
6115
6116#[cfg(test)]
6117#[path = "runtime/codex_v2_lifecycle_tests.rs"]
6118mod codex_v2_lifecycle_tests;
6119
6120#[cfg(test)]
6121mod tests {
6122    use super::*;
6123    use futures::stream;
6124    use roder_api::catalog::{
6125        PROVIDER_MOCK, REASONING_HIGH, REASONING_LOW, REASONING_MEDIUM, REASONING_MINIMAL,
6126        REASONING_NONE, REASONING_XHIGH,
6127    };
6128    use roder_api::extension::ExtensionRegistryBuilder;
6129    use roder_api::inference::{
6130        CompletionMetadata, InferenceCapabilities, InferenceEngine, InferenceEventStream,
6131        InferenceProviderContext, InferenceTurnContext, MessageDelta, ModelDescriptor,
6132        ModelInstructionOverlay, ModelProfileReasoning, ModelSchemaPolicy, ProviderFamily,
6133        ReasoningEffortDescriptor,
6134    };
6135    use roder_api::inference_routing::{
6136        InferenceRouter, InferenceRoutingContext, InferenceRoutingDecision, InferenceRoutingOutcome,
6137    };
6138    use roder_api::thread::ThreadStoreFactory;
6139    use roder_api::tools::{ToolContributor, ToolExecutor, ToolSpec};
6140    use roder_ext_jsonl_thread_store::store::JsonlThreadStoreFactory;
6141    use std::sync::Mutex as StdMutex;
6142
6143    fn test_workspace() -> String {
6144        std::env::current_dir().unwrap().display().to_string()
6145    }
6146
6147    #[test]
6148    fn interactive_explicit_turn_deadline_has_bounded_remaining_time() {
6149        let config = RuntimeConfig {
6150            runtime_profile: RuntimeProfile::Interactive,
6151            turn_deadline_seconds: Some(60),
6152            ..RuntimeConfig::default()
6153        };
6154
6155        let deadline = turn_deadline_for_config(&config).expect("configured turn deadline");
6156        let remaining = deadline_remaining_seconds(Some(deadline));
6157
6158        assert!(
6159            matches!(remaining, Some(1..=60)),
6160            "interactive deadline must be positive and no longer than configured: {remaining:?}"
6161        );
6162    }
6163
6164    #[test]
6165    fn interactive_without_turn_deadline_stays_unbounded() {
6166        let config = RuntimeConfig {
6167            runtime_profile: RuntimeProfile::Interactive,
6168            turn_deadline_seconds: None,
6169            ..RuntimeConfig::default()
6170        };
6171
6172        assert_eq!(turn_deadline_for_config(&config), None);
6173    }
6174
6175    struct MetadataMissingStore;
6176
6177    #[async_trait::async_trait]
6178    impl ThreadStore for MetadataMissingStore {
6179        fn id(&self) -> roder_api::thread::ThreadStoreId {
6180            "metadata-missing-store".to_string()
6181        }
6182
6183        async fn create_thread(&self, metadata: ThreadMetadata) -> anyhow::Result<ThreadMetadata> {
6184            Ok(metadata)
6185        }
6186
6187        async fn list_threads(&self) -> anyhow::Result<Vec<ThreadMetadata>> {
6188            Ok(Vec::new())
6189        }
6190
6191        async fn load_thread(
6192            &self,
6193            _thread_id: &ThreadId,
6194        ) -> anyhow::Result<Option<ThreadSnapshot>> {
6195            Ok(Some(ThreadSnapshot {
6196                metadata: None,
6197                ..ThreadSnapshot::default()
6198            }))
6199        }
6200
6201        async fn append_event(
6202            &self,
6203            _thread_id: &ThreadId,
6204            _envelope: &EventEnvelope,
6205        ) -> anyhow::Result<()> {
6206            Ok(())
6207        }
6208    }
6209
6210    struct MetadataMissingStoreFactory;
6211
6212    impl ThreadStoreFactory for MetadataMissingStoreFactory {
6213        fn id(&self) -> roder_api::thread::ThreadStoreId {
6214            "metadata-missing-store".to_string()
6215        }
6216
6217        fn create(&self) -> Arc<dyn ThreadStore> {
6218            Arc::new(MetadataMissingStore)
6219        }
6220    }
6221
6222    #[test]
6223    fn synthetic_app_server_events_are_not_thread_events() {
6224        for thread_id in ["app-server", "runtime", "thread-workflow"] {
6225            assert!(!should_persist_thread_event(thread_id));
6226        }
6227        assert!(should_persist_thread_event("thread-discovery"));
6228        assert!(should_persist_thread_event("thread-plan"));
6229        assert!(should_persist_thread_event("thread-process"));
6230        assert!(should_persist_thread_event("thread-1"));
6231    }
6232
6233    #[test]
6234    fn server_side_compaction_uses_catalog_ninety_percent_default() {
6235        assert_eq!(
6236            server_side_compaction_threshold(&RuntimeConfig::default(), "gpt-5.5"),
6237            Some(945_000)
6238        );
6239        assert_eq!(
6240            server_side_compaction_threshold(&RuntimeConfig::default(), "gpt-5.3-codex-spark"),
6241            Some(115_200)
6242        );
6243    }
6244
6245    #[test]
6246    fn server_side_compaction_respects_explicit_config_override() {
6247        let cfg = RuntimeConfig {
6248            auto_compact_token_limit: Some(123_456),
6249            ..RuntimeConfig::default()
6250        };
6251
6252        assert_eq!(
6253            server_side_compaction_threshold(&cfg, "gpt-5.5"),
6254            Some(123_456)
6255        );
6256    }
6257
6258    #[tokio::test]
6259    async fn pre_request_compaction_runs_when_server_side_model_is_at_context_window() {
6260        let captured = Arc::new(StdMutex::new(None));
6261        let mut builder = ExtensionRegistryBuilder::new();
6262        builder.inference_engine(Arc::new(CapturingEngine {
6263            request: captured.clone(),
6264        }));
6265        let thread_root = std::env::temp_dir().join(format!(
6266            "roder-pre-request-compaction-{}",
6267            uuid::Uuid::new_v4()
6268        ));
6269        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
6270            base_path: thread_root.clone(),
6271        }));
6272        let runtime = Arc::new(
6273            Runtime::new(
6274                builder.build().unwrap(),
6275                RuntimeConfig {
6276                    default_provider: PROVIDER_MOCK.to_string(),
6277                    default_model: "gpt-5.5".to_string(),
6278                    file_backed_dynamic_context: true,
6279                    ..RuntimeConfig::default()
6280                },
6281            )
6282            .unwrap(),
6283        );
6284        let thread_id = runtime
6285            .create_thread(Some("Pre-request compaction".to_string()))
6286            .await
6287            .unwrap()
6288            .thread_id;
6289        let old_turn = "old-turn".to_string();
6290        runtime
6291            .persist_turn_item(
6292                &thread_id,
6293                &old_turn,
6294                &TranscriptItem::UserMessage(UserMessage::text("old context ".repeat(4_300_000))),
6295            )
6296            .await
6297            .unwrap();
6298
6299        let mut events = runtime.subscribe_events();
6300        runtime
6301            .start_turn(StartTurnRequest {
6302                thread_id: thread_id.clone(),
6303                message: "continue".to_string(),
6304                images: Vec::new(),
6305                provider_override: None,
6306                model_override: None,
6307                reasoning_override: None,
6308                workspace: test_workspace(),
6309                instructions: InstructionBundle::default(),
6310                developer_context: None,
6311                task_ledger_required: false,
6312                service_tier_override: None,
6313            })
6314            .await
6315            .unwrap();
6316        loop {
6317            let envelope = tokio::time::timeout(std::time::Duration::from_secs(5), events.recv())
6318                .await
6319                .unwrap()
6320                .unwrap();
6321            if envelope.thread_id.as_deref() == Some(&thread_id)
6322                && matches!(envelope.event, RoderEvent::TurnCompleted(_))
6323            {
6324                break;
6325            }
6326        }
6327
6328        let request = captured.lock().unwrap().clone().unwrap();
6329        assert!(
6330            matches!(
6331                request.transcript.first(),
6332                Some(TranscriptItem::ContextCompaction(_))
6333            ),
6334            "provider request should start with a local emergency compaction item"
6335        );
6336        assert!(
6337            request.transcript.len() < 4,
6338            "provider request should not replay the full oversized prior transcript: {:?}",
6339            request.transcript
6340        );
6341
6342        let _ = std::fs::remove_dir_all(thread_root);
6343    }
6344
6345    #[tokio::test]
6346    async fn continue_after_context_window_failure_compacts_before_provider_request() {
6347        let captured = Arc::new(StdMutex::new(None));
6348        let mut builder = ExtensionRegistryBuilder::new();
6349        builder.inference_engine(Arc::new(CapturingEngine {
6350            request: captured.clone(),
6351        }));
6352        let thread_root = std::env::temp_dir().join(format!(
6353            "roder-context-failure-continue-{}",
6354            uuid::Uuid::new_v4()
6355        ));
6356        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
6357            base_path: thread_root.clone(),
6358        }));
6359        let runtime = Arc::new(
6360            Runtime::new(
6361                builder.build().unwrap(),
6362                RuntimeConfig {
6363                    default_provider: PROVIDER_MOCK.to_string(),
6364                    default_model: "gpt-5.5".to_string(),
6365                    file_backed_dynamic_context: true,
6366                    ..RuntimeConfig::default()
6367                },
6368            )
6369            .unwrap(),
6370        );
6371        let thread_id = runtime
6372            .create_thread(Some("Context failure continue".to_string()))
6373            .await
6374            .unwrap()
6375            .thread_id;
6376        let failed_turn = "failed-turn".to_string();
6377        runtime
6378            .persist_turn_item(
6379                &thread_id,
6380                &failed_turn,
6381                &TranscriptItem::UserMessage(UserMessage::text("old work ".repeat(10_000))),
6382            )
6383            .await
6384            .unwrap();
6385        runtime
6386            .persist_turn_item(
6387                &thread_id,
6388                &failed_turn,
6389                &TranscriptItem::Error(ErrorRecord {
6390                    message: "Your input exceeds the context window of this model. Please adjust your input and try again."
6391                        .to_string(),
6392                }),
6393            )
6394            .await
6395            .unwrap();
6396
6397        let mut events = runtime.subscribe_events();
6398        runtime
6399            .start_turn(StartTurnRequest {
6400                thread_id: thread_id.clone(),
6401                message: "continue".to_string(),
6402                images: Vec::new(),
6403                provider_override: None,
6404                model_override: None,
6405                reasoning_override: None,
6406                workspace: test_workspace(),
6407                instructions: InstructionBundle::default(),
6408                developer_context: None,
6409                task_ledger_required: false,
6410                service_tier_override: None,
6411            })
6412            .await
6413            .unwrap();
6414        loop {
6415            let envelope = tokio::time::timeout(std::time::Duration::from_secs(5), events.recv())
6416                .await
6417                .unwrap()
6418                .unwrap();
6419            if envelope.thread_id.as_deref() == Some(&thread_id)
6420                && matches!(envelope.event, RoderEvent::TurnCompleted(_))
6421            {
6422                break;
6423            }
6424        }
6425
6426        let request = captured.lock().unwrap().clone().unwrap();
6427        assert!(
6428            matches!(
6429                request.transcript.first(),
6430                Some(TranscriptItem::ContextCompaction(_))
6431            ),
6432            "provider request after context-window failure should start with local compaction"
6433        );
6434        assert!(
6435            request
6436                .transcript
6437                .iter()
6438                .any(|item| matches!(item, TranscriptItem::UserMessage(message) if message.text == "continue")),
6439            "current continue prompt must be preserved: {:?}",
6440            request.transcript
6441        );
6442        assert!(
6443            !request.transcript.iter().any(
6444                |item| matches!(item, TranscriptItem::Error(error) if error.message.contains("context window"))
6445            ),
6446            "raw prior context-window error should be summarized, not replayed: {:?}",
6447            request.transcript
6448        );
6449
6450        let _ = std::fs::remove_dir_all(thread_root);
6451    }
6452
6453    #[tokio::test]
6454    async fn workspace_for_thread_falls_back_when_metadata_is_missing() {
6455        let workspace = test_workspace();
6456        let mut builder = ExtensionRegistryBuilder::new();
6457        builder.inference_engine(Arc::new(FakeInferenceEngine));
6458        builder.thread_store_factory(Arc::new(MetadataMissingStoreFactory));
6459        let runtime = Runtime::new(
6460            builder.build().unwrap(),
6461            RuntimeConfig {
6462                workspace: Some(workspace.clone()),
6463                ..RuntimeConfig::default()
6464            },
6465        )
6466        .unwrap();
6467
6468        let resolved = runtime
6469            .workspace_for_thread(&ThreadId::from("thread-workflow"))
6470            .await
6471            .unwrap();
6472
6473        assert_eq!(resolved, workspace);
6474    }
6475
6476    #[tokio::test]
6477    async fn automations_can_create_project_thread_with_model_overrides() {
6478        let runtime = Runtime::fake().unwrap();
6479        let workspace = std::env::temp_dir().join("project");
6480        let metadata = runtime
6481            .create_thread_with(CreateThreadRequest {
6482                title: Some("Automation: nightly status".to_string()),
6483                workspace: workspace.display().to_string(),
6484                workspace_id: None,
6485                root_id: None,
6486                provider: Some("mock".to_string()),
6487                model: Some("mock".to_string()),
6488                selection_mode: None,
6489                tool_allowlist: Vec::new(),
6490                service_tier: None,
6491                developer_instructions: None,
6492                external_tools: Vec::new(),
6493                runner: None,
6494            })
6495            .await
6496            .unwrap();
6497
6498        assert_eq!(
6499            metadata.title.as_deref(),
6500            Some("Automation: nightly status")
6501        );
6502        assert_eq!(metadata.workspace, workspace.display().to_string());
6503        assert_eq!(metadata.provider.as_deref(), Some("mock"));
6504        assert_eq!(metadata.model.as_deref(), Some("mock"));
6505    }
6506
6507    #[tokio::test]
6508    async fn prior_provider_compaction_boundary_is_used_on_next_turn() {
6509        let captured = Arc::new(StdMutex::new(None));
6510        let mut builder = ExtensionRegistryBuilder::new();
6511        builder.inference_engine(Arc::new(CapturingEngine {
6512            request: captured.clone(),
6513        }));
6514        let thread_root = std::env::temp_dir().join(format!(
6515            "roder-provider-compaction-boundary-{}",
6516            uuid::Uuid::new_v4()
6517        ));
6518        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
6519            base_path: thread_root.clone(),
6520        }));
6521        let runtime = Arc::new(
6522            Runtime::new(
6523                builder.build().unwrap(),
6524                RuntimeConfig {
6525                    default_provider: PROVIDER_MOCK.to_string(),
6526                    default_model: "gpt-5.5".to_string(),
6527                    ..RuntimeConfig::default()
6528                },
6529            )
6530            .unwrap(),
6531        );
6532        let thread_id = runtime
6533            .create_thread(Some("Provider compaction boundary".to_string()))
6534            .await
6535            .unwrap()
6536            .thread_id;
6537        let old_turn = "old-turn".to_string();
6538        runtime
6539            .persist_turn_item(
6540                &thread_id,
6541                &old_turn,
6542                &TranscriptItem::UserMessage(UserMessage::text(
6543                    "old history that must not be replayed after provider compaction",
6544                )),
6545            )
6546            .await
6547            .unwrap();
6548        runtime
6549            .persist_turn_item(
6550                &thread_id,
6551                &old_turn,
6552                &TranscriptItem::AssistantMessage(AssistantMessage {
6553                    text: "old answer".to_string(),
6554                    phase: None,
6555                }),
6556            )
6557            .await
6558            .unwrap();
6559        runtime
6560            .persist_turn_item(
6561                &thread_id,
6562                &old_turn,
6563                &TranscriptItem::ProviderMetadata(serde_json::json!({
6564                    "output": [{
6565                        "id": "cmp_1",
6566                        "type": "compaction",
6567                        "encrypted_content": "opaque-state"
6568                    }]
6569                })),
6570            )
6571            .await
6572            .unwrap();
6573        runtime
6574            .persist_turn_item(
6575                &thread_id,
6576                &old_turn,
6577                &TranscriptItem::AssistantMessage(AssistantMessage {
6578                    text: "after compact".to_string(),
6579                    phase: None,
6580                }),
6581            )
6582            .await
6583            .unwrap();
6584
6585        let mut events = runtime.subscribe_events();
6586        runtime
6587            .start_turn(StartTurnRequest {
6588                thread_id: thread_id.clone(),
6589                message: "continue".to_string(),
6590                images: Vec::new(),
6591                provider_override: None,
6592                model_override: None,
6593                reasoning_override: None,
6594                workspace: test_workspace(),
6595                instructions: InstructionBundle::default(),
6596                developer_context: None,
6597                task_ledger_required: false,
6598                service_tier_override: None,
6599            })
6600            .await
6601            .unwrap();
6602        loop {
6603            let envelope = tokio::time::timeout(std::time::Duration::from_secs(5), events.recv())
6604                .await
6605                .unwrap()
6606                .unwrap();
6607            if envelope.thread_id.as_deref() == Some(&thread_id)
6608                && matches!(envelope.event, RoderEvent::TurnCompleted(_))
6609            {
6610                break;
6611            }
6612        }
6613
6614        let request = captured.lock().unwrap().clone().unwrap();
6615        let compaction_idx = request.transcript.iter().position(|item| {
6616            matches!(
6617                item,
6618                TranscriptItem::ProviderMetadata(metadata)
6619                    if crate::compaction::provider_metadata_has_compaction(metadata)
6620            )
6621        });
6622        assert!(
6623            compaction_idx.is_some(),
6624            "next provider request should include the provider compaction item: {:?}",
6625            request.transcript
6626        );
6627        // Skill/context injectors may prepend non-history items, but no prior-turn
6628        // conversation may appear before the latest provider compaction boundary.
6629        let history_before_boundary =
6630            request
6631                .transcript
6632                .iter()
6633                .take(compaction_idx.unwrap())
6634                .any(|item| match item {
6635                    TranscriptItem::AssistantMessage(_)
6636                    | TranscriptItem::ToolCall(_)
6637                    | TranscriptItem::ToolResult(_) => true,
6638                    TranscriptItem::UserMessage(message) => {
6639                        !message.text.contains("<skills>") && message.text != "continue"
6640                    }
6641                    _ => false,
6642                });
6643        assert!(
6644            !history_before_boundary,
6645            "pre-provider-compaction conversation must not be replayed: {:?}",
6646            request.transcript
6647        );
6648        assert!(
6649            !request.transcript.iter().any(|item| {
6650                matches!(
6651                    item,
6652                    TranscriptItem::UserMessage(message)
6653                        if message.text.contains("old history that must not be replayed")
6654                )
6655            }),
6656            "pre-provider-compaction history must not be replayed: {:?}",
6657            request.transcript
6658        );
6659        assert!(
6660            request.transcript.iter().any(|item| {
6661                matches!(
6662                    item,
6663                    TranscriptItem::UserMessage(message) if message.text == "continue"
6664                )
6665            }),
6666            "current continue prompt must be preserved: {:?}",
6667            request.transcript
6668        );
6669        assert_eq!(
6670            request.runtime.auto_compact_token_limit,
6671            Some(945_000),
6672            "gpt-5.5 should keep server-side auto compaction configured"
6673        );
6674
6675        let _ = std::fs::remove_dir_all(thread_root);
6676    }
6677
6678    #[test]
6679    fn server_side_compaction_is_only_enabled_for_supported_models() {
6680        let cfg = RuntimeConfig {
6681            auto_compact_token_limit: Some(123_456),
6682            ..RuntimeConfig::default()
6683        };
6684
6685        assert_eq!(server_side_compaction_threshold(&cfg, "mock"), None);
6686        assert_eq!(
6687            server_side_compaction_threshold(&cfg, "codex-auto-review"),
6688            None
6689        );
6690    }
6691
6692    #[test]
6693    fn reasoning_is_disabled_for_models_without_reasoning_support() {
6694        let cfg = RuntimeConfig {
6695            reasoning: Some(REASONING_HIGH.to_string()),
6696            ..RuntimeConfig::default()
6697        };
6698
6699        assert_eq!(
6700            effective_reasoning_for_model(&cfg, "claude-haiku-4-5-20251001"),
6701            REASONING_NONE
6702        );
6703        assert_eq!(
6704            reasoning_for_model(&cfg, "claude-haiku-4-5-20251001"),
6705            ReasoningConfig::default()
6706        );
6707    }
6708
6709    #[test]
6710    fn unsupported_configured_reasoning_falls_back_to_model_default() {
6711        let cfg = RuntimeConfig {
6712            reasoning: Some(REASONING_MINIMAL.to_string()),
6713            ..RuntimeConfig::default()
6714        };
6715
6716        assert_eq!(
6717            effective_reasoning_for_model(&cfg, "gpt-5.5"),
6718            REASONING_MEDIUM
6719        );
6720    }
6721
6722    #[test]
6723    fn unsupported_configured_gemini_reasoning_is_rejected() {
6724        let mut builder = ExtensionRegistryBuilder::new();
6725        builder.inference_engine(std::sync::Arc::new(FakeInferenceEngine));
6726
6727        let err = match Runtime::new(
6728            builder.build().unwrap(),
6729            RuntimeConfig {
6730                default_model: "gemini-3.5-flash".to_string(),
6731                reasoning: Some(REASONING_XHIGH.to_string()),
6732                ..RuntimeConfig::default()
6733            },
6734        ) {
6735            Ok(_) => panic!("expected unsupported Gemini reasoning to be rejected"),
6736            Err(err) => err,
6737        };
6738
6739        assert!(
6740            err.to_string()
6741                .contains("model gemini-3.5-flash does not support reasoning effort xhigh")
6742        );
6743    }
6744
6745    #[tokio::test]
6746    async fn selecting_none_for_non_reasoning_model_preserves_stored_preference() {
6747        let runtime = Runtime::new(
6748            Runtime::fake().unwrap().registry,
6749            RuntimeConfig {
6750                reasoning: Some(REASONING_HIGH.to_string()),
6751                ..RuntimeConfig::default()
6752            },
6753        )
6754        .unwrap();
6755
6756        let cfg = runtime
6757            .select_provider(
6758                roder_api::catalog::PROVIDER_MOCK.to_string(),
6759                Some("claude-haiku-4-5-20251001".to_string()),
6760                Some(REASONING_NONE.to_string()),
6761            )
6762            .await
6763            .unwrap();
6764
6765        assert_eq!(cfg.reasoning.as_deref(), Some(REASONING_HIGH));
6766        assert_eq!(runtime.effective_reasoning().await, REASONING_NONE);
6767    }
6768
6769    #[tokio::test]
6770    async fn selecting_none_for_model_that_supports_none_updates_preference() {
6771        let runtime = Runtime::new(
6772            Runtime::fake().unwrap().registry,
6773            RuntimeConfig {
6774                reasoning: Some(REASONING_HIGH.to_string()),
6775                ..RuntimeConfig::default()
6776            },
6777        )
6778        .unwrap();
6779
6780        let cfg = runtime
6781            .select_provider(
6782                roder_api::catalog::PROVIDER_MOCK.to_string(),
6783                Some("mock".to_string()),
6784                Some(REASONING_NONE.to_string()),
6785            )
6786            .await
6787            .unwrap();
6788
6789        assert_eq!(cfg.reasoning.as_deref(), Some(REASONING_NONE));
6790    }
6791
6792    #[test]
6793    fn parallel_tool_calls_default_on_with_model_override() {
6794        assert!(parallel_tool_calls_for_model(
6795            &RuntimeConfig::default(),
6796            "custom-model"
6797        ));
6798
6799        let cfg = RuntimeConfig {
6800            model_parallel_tool_calls: std::collections::HashMap::from([(
6801                "custom-model".to_string(),
6802                false,
6803            )]),
6804            ..RuntimeConfig::default()
6805        };
6806
6807        assert!(!parallel_tool_calls_for_model(&cfg, "custom-model"));
6808        assert!(parallel_tool_calls_for_model(&cfg, "other-model"));
6809    }
6810
6811    #[test]
6812    fn profile_parallel_tool_calls_applies_between_config_and_default() {
6813        let cfg = RuntimeConfig {
6814            model_profiles: std::collections::HashMap::from([(
6815                "gpt-5.5".to_string(),
6816                test_model_profile("gpt-5.5"),
6817            )]),
6818            ..RuntimeConfig::default()
6819        };
6820
6821        assert!(!parallel_tool_calls_for_model(&cfg, "gpt-5.5"));
6822
6823        let cfg = RuntimeConfig {
6824            model_parallel_tool_calls: std::collections::HashMap::from([(
6825                "gpt-5.5".to_string(),
6826                true,
6827            )]),
6828            ..cfg
6829        };
6830
6831        assert!(parallel_tool_calls_for_model(&cfg, "gpt-5.5"));
6832    }
6833
6834    struct CapturingEngine {
6835        request: Arc<StdMutex<Option<AgentInferenceRequest>>>,
6836    }
6837
6838    #[async_trait::async_trait]
6839    impl InferenceEngine for CapturingEngine {
6840        fn id(&self) -> String {
6841            roder_api::catalog::PROVIDER_MOCK.to_string()
6842        }
6843
6844        fn capabilities(&self) -> InferenceCapabilities {
6845            InferenceCapabilities::coding_agent_default()
6846        }
6847
6848        async fn list_models(
6849            &self,
6850            _ctx: InferenceProviderContext<'_>,
6851        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
6852            Ok(roder_api::catalog::models_for_provider(
6853                roder_api::catalog::PROVIDER_MOCK,
6854                true,
6855            ))
6856        }
6857
6858        async fn stream_turn(
6859            &self,
6860            _ctx: InferenceTurnContext<'_>,
6861            request: AgentInferenceRequest,
6862        ) -> anyhow::Result<InferenceEventStream> {
6863            *self.request.lock().unwrap() = Some(request);
6864            Ok(Box::pin(stream::iter(vec![
6865                Ok(InferenceEvent::MessageDelta(MessageDelta {
6866                    text: "done".to_string(),
6867                    phase: None,
6868                })),
6869                Ok(InferenceEvent::Completed(CompletionMetadata {
6870                    stop_reason: Some("stop".to_string()),
6871                    provider_response_id: None,
6872                })),
6873            ])))
6874        }
6875    }
6876
6877    struct RoutingCaptureEngine {
6878        id: &'static str,
6879        models: Vec<ModelDescriptor>,
6880        requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
6881    }
6882
6883    #[async_trait::async_trait]
6884    impl InferenceEngine for RoutingCaptureEngine {
6885        fn id(&self) -> String {
6886            self.id.to_string()
6887        }
6888
6889        fn capabilities(&self) -> InferenceCapabilities {
6890            InferenceCapabilities::coding_agent_default()
6891        }
6892
6893        async fn list_models(
6894            &self,
6895            _ctx: InferenceProviderContext<'_>,
6896        ) -> anyhow::Result<Vec<ModelDescriptor>> {
6897            Ok(self.models.clone())
6898        }
6899
6900        async fn stream_turn(
6901            &self,
6902            _ctx: InferenceTurnContext<'_>,
6903            request: AgentInferenceRequest,
6904        ) -> anyhow::Result<InferenceEventStream> {
6905            self.requests.lock().unwrap().push(request);
6906            Ok(Box::pin(stream::iter(vec![
6907                Ok(InferenceEvent::MessageDelta(MessageDelta {
6908                    text: "routed".to_string(),
6909                    phase: None,
6910                })),
6911                Ok(InferenceEvent::Completed(CompletionMetadata {
6912                    stop_reason: Some("stop".to_string()),
6913                    provider_response_id: None,
6914                })),
6915            ])))
6916        }
6917    }
6918
6919    struct StaticRouter {
6920        id: &'static str,
6921        decision: InferenceRoutingDecision,
6922        contexts: Arc<StdMutex<Vec<InferenceRoutingContext>>>,
6923    }
6924
6925    #[async_trait::async_trait]
6926    impl InferenceRouter for StaticRouter {
6927        fn id(&self) -> String {
6928            self.id.to_string()
6929        }
6930
6931        async fn route(
6932            &self,
6933            context: InferenceRoutingContext,
6934        ) -> anyhow::Result<InferenceRoutingDecision> {
6935            self.contexts.lock().unwrap().push(context);
6936            Ok(self.decision.clone())
6937        }
6938    }
6939
6940    fn routing_test_model(id: &str, supported_reasoning: &[&str]) -> ModelDescriptor {
6941        ModelDescriptor {
6942            id: id.to_string(),
6943            name: id.to_string(),
6944            context_window: Some(128_000),
6945            default_reasoning: supported_reasoning
6946                .first()
6947                .map(|effort| (*effort).to_string()),
6948            supported_reasoning: supported_reasoning
6949                .iter()
6950                .map(|effort| ReasoningEffortDescriptor {
6951                    effort: (*effort).to_string(),
6952                    description: format!("{effort} reasoning"),
6953                })
6954                .collect(),
6955        }
6956    }
6957
6958    struct TaskLedgerCompletionGateEngine {
6959        calls: StdMutex<u32>,
6960        requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
6961    }
6962
6963    #[async_trait::async_trait]
6964    impl InferenceEngine for TaskLedgerCompletionGateEngine {
6965        fn id(&self) -> String {
6966            roder_api::catalog::PROVIDER_MOCK.to_string()
6967        }
6968
6969        fn capabilities(&self) -> InferenceCapabilities {
6970            InferenceCapabilities::coding_agent_default()
6971        }
6972
6973        async fn list_models(
6974            &self,
6975            _ctx: InferenceProviderContext<'_>,
6976        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
6977            Ok(roder_api::catalog::models_for_provider(
6978                roder_api::catalog::PROVIDER_MOCK,
6979                true,
6980            ))
6981        }
6982
6983        async fn stream_turn(
6984            &self,
6985            _ctx: InferenceTurnContext<'_>,
6986            request: AgentInferenceRequest,
6987        ) -> anyhow::Result<InferenceEventStream> {
6988            self.requests.lock().unwrap().push(request);
6989            let mut calls = self.calls.lock().unwrap();
6990            *calls += 1;
6991            let events = match *calls {
6992                1 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
6993                    id: "ledger-open".to_string(),
6994                    name: TASK_LEDGER_TOOL_NAME.to_string(),
6995                    arguments: serde_json::json!({
6996                        "tasks": [
6997                            {
6998                                "id": "inspect",
6999                                "content": "Inspect local assets",
7000                                "status": "completed",
7001                                "evidence": "listed workspace"
7002                            },
7003                            {
7004                                "id": "write",
7005                                "content": "Write /app/result.txt",
7006                                "status": "pending"
7007                            }
7008                        ],
7009                        "requireCompletionEvidence": true
7010                    })
7011                    .to_string(),
7012                }))],
7013                3 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7014                    id: "ledger-complete".to_string(),
7015                    name: TASK_LEDGER_TOOL_NAME.to_string(),
7016                    arguments: serde_json::json!({
7017                        "tasks": [
7018                            {
7019                                "id": "inspect",
7020                                "content": "Inspect local assets",
7021                                "status": "completed",
7022                                "evidence": "listed workspace"
7023                            },
7024                            {
7025                                "id": "write",
7026                                "content": "Write /app/result.txt",
7027                                "status": "completed",
7028                                "evidence": "wrote answer"
7029                            }
7030                        ],
7031                        "requireCompletionEvidence": true
7032                    })
7033                    .to_string(),
7034                }))],
7035                _ => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
7036                    text: "final".to_string(),
7037                    phase: None,
7038                }))],
7039            };
7040            Ok(Box::pin(stream::iter(events.into_iter().chain(
7041                std::iter::once(Ok(InferenceEvent::Completed(CompletionMetadata {
7042                    stop_reason: Some("stop".to_string()),
7043                    provider_response_id: None,
7044                }))),
7045            ))))
7046        }
7047    }
7048
7049    struct VerificationGateEngine {
7050        calls: StdMutex<u32>,
7051    }
7052
7053    #[async_trait::async_trait]
7054    impl InferenceEngine for VerificationGateEngine {
7055        fn id(&self) -> String {
7056            roder_api::catalog::PROVIDER_MOCK.to_string()
7057        }
7058
7059        fn capabilities(&self) -> InferenceCapabilities {
7060            InferenceCapabilities::coding_agent_default()
7061        }
7062
7063        async fn list_models(
7064            &self,
7065            _ctx: InferenceProviderContext<'_>,
7066        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7067            Ok(roder_api::catalog::models_for_provider(
7068                roder_api::catalog::PROVIDER_MOCK,
7069                true,
7070            ))
7071        }
7072
7073        async fn stream_turn(
7074            &self,
7075            _ctx: InferenceTurnContext<'_>,
7076            request: AgentInferenceRequest,
7077        ) -> anyhow::Result<InferenceEventStream> {
7078            let mut calls = self.calls.lock().unwrap();
7079            *calls += 1;
7080            let events = match *calls {
7081                1 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7082                    id: "write-1".to_string(),
7083                    name: "write_file".to_string(),
7084                    arguments: serde_json::json!({
7085                        "path": "src/lib.rs",
7086                        "content": "pub fn answer() -> u8 { 42 }\n"
7087                    })
7088                    .to_string(),
7089                }))],
7090                2 => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
7091                    text: "done too early".to_string(),
7092                    phase: None,
7093                }))],
7094                3 if request.transcript.iter().any(|item| {
7095                    matches!(
7096                        item,
7097                        TranscriptItem::UserMessage(message)
7098                            if message.text.contains("Verification gate blocked final completion")
7099                    )
7100                }) =>
7101                {
7102                    vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7103                        id: "verify-1".to_string(),
7104                        name: crate::verification_gate::VERIFICATION_TOOL_NAME.to_string(),
7105                        arguments: serde_json::json!({
7106                            "originalTask": "write code",
7107                            "changedFiles": ["src/lib.rs"],
7108                            "toolEvidence": ["write_file wrote src/lib.rs"],
7109                            "testsRun": ["cargo test -p roder-core verification_gate"],
7110                            "openGaps": [],
7111                            "status": "completed"
7112                        })
7113                        .to_string(),
7114                    }))]
7115                }
7116                _ => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
7117                    text: "verified final".to_string(),
7118                    phase: None,
7119                }))],
7120            };
7121            Ok(Box::pin(stream::iter(events.into_iter().chain(
7122                std::iter::once(Ok(InferenceEvent::Completed(CompletionMetadata {
7123                    stop_reason: Some("stop".to_string()),
7124                    provider_response_id: None,
7125                }))),
7126            ))))
7127        }
7128    }
7129
7130    struct SpeedPolicyEngine {
7131        calls: StdMutex<u32>,
7132        requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
7133    }
7134
7135    #[async_trait::async_trait]
7136    impl InferenceEngine for SpeedPolicyEngine {
7137        fn id(&self) -> String {
7138            roder_api::catalog::PROVIDER_MOCK.to_string()
7139        }
7140
7141        fn capabilities(&self) -> InferenceCapabilities {
7142            InferenceCapabilities::coding_agent_default()
7143        }
7144
7145        async fn list_models(
7146            &self,
7147            _ctx: InferenceProviderContext<'_>,
7148        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7149            Ok(roder_api::catalog::models_for_provider(
7150                roder_api::catalog::PROVIDER_MOCK,
7151                true,
7152            ))
7153        }
7154
7155        async fn stream_turn(
7156            &self,
7157            _ctx: InferenceTurnContext<'_>,
7158            request: AgentInferenceRequest,
7159        ) -> anyhow::Result<InferenceEventStream> {
7160            self.requests.lock().unwrap().push(request.clone());
7161            let mut calls = self.calls.lock().unwrap();
7162            *calls += 1;
7163            let events = match *calls {
7164                1 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7165                    id: "write-1".to_string(),
7166                    name: "write_file".to_string(),
7167                    arguments: serde_json::json!({
7168                        "path": "src/lib.rs",
7169                        "content": "pub fn answer() -> u8 { 42 }\n"
7170                    })
7171                    .to_string(),
7172                }))],
7173                3 if request.transcript.iter().any(|item| {
7174                    matches!(
7175                        item,
7176                        TranscriptItem::UserMessage(message)
7177                            if message.text.contains("Verification gate blocked final completion")
7178                    )
7179                }) =>
7180                {
7181                    vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7182                        id: "verify-1".to_string(),
7183                        name: crate::verification_gate::VERIFICATION_TOOL_NAME.to_string(),
7184                        arguments: serde_json::json!({
7185                            "originalTask": "write code",
7186                            "changedFiles": ["src/lib.rs"],
7187                            "toolEvidence": ["write_file wrote src/lib.rs"],
7188                            "testsRun": ["cargo test -p roder-core speed_policy"],
7189                            "openGaps": [],
7190                            "status": "completed"
7191                        })
7192                        .to_string(),
7193                    }))]
7194                }
7195                _ => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
7196                    text: "done".to_string(),
7197                    phase: None,
7198                }))],
7199            };
7200            Ok(Box::pin(stream::iter(events.into_iter().chain(
7201                std::iter::once(Ok(InferenceEvent::Completed(CompletionMetadata {
7202                    stop_reason: Some("stop".to_string()),
7203                    provider_response_id: None,
7204                }))),
7205            ))))
7206        }
7207    }
7208
7209    struct SwitchCaptureEngine {
7210        requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
7211    }
7212
7213    #[async_trait::async_trait]
7214    impl InferenceEngine for SwitchCaptureEngine {
7215        fn id(&self) -> String {
7216            roder_api::catalog::PROVIDER_MOCK.to_string()
7217        }
7218
7219        fn capabilities(&self) -> InferenceCapabilities {
7220            InferenceCapabilities::coding_agent_default()
7221        }
7222
7223        async fn list_models(
7224            &self,
7225            _ctx: InferenceProviderContext<'_>,
7226        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7227            Ok(roder_api::catalog::models_for_provider(
7228                roder_api::catalog::PROVIDER_MOCK,
7229                true,
7230            ))
7231        }
7232
7233        async fn stream_turn(
7234            &self,
7235            _ctx: InferenceTurnContext<'_>,
7236            request: AgentInferenceRequest,
7237        ) -> anyhow::Result<InferenceEventStream> {
7238            self.requests.lock().unwrap().push(request);
7239            Ok(Box::pin(stream::iter(vec![
7240                Ok(InferenceEvent::MessageDelta(MessageDelta {
7241                    text: "done".to_string(),
7242                    phase: None,
7243                })),
7244                Ok(InferenceEvent::Completed(CompletionMetadata {
7245                    stop_reason: Some("stop".to_string()),
7246                    provider_response_id: None,
7247                })),
7248            ])))
7249        }
7250    }
7251
7252    struct DeadlineEngine;
7253
7254    #[async_trait::async_trait]
7255    impl InferenceEngine for DeadlineEngine {
7256        fn id(&self) -> String {
7257            roder_api::catalog::PROVIDER_MOCK.to_string()
7258        }
7259
7260        fn capabilities(&self) -> InferenceCapabilities {
7261            InferenceCapabilities::coding_agent_default()
7262        }
7263
7264        async fn list_models(
7265            &self,
7266            _ctx: InferenceProviderContext<'_>,
7267        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7268            Ok(Vec::new())
7269        }
7270
7271        async fn stream_turn(
7272            &self,
7273            _ctx: InferenceTurnContext<'_>,
7274            _request: AgentInferenceRequest,
7275        ) -> anyhow::Result<InferenceEventStream> {
7276            Ok(Box::pin(stream::once(async {
7277                tokio::time::sleep(std::time::Duration::from_secs(60)).await;
7278                Ok(InferenceEvent::MessageDelta(MessageDelta {
7279                    text: "too late".to_string(),
7280                    phase: None,
7281                }))
7282            })))
7283        }
7284    }
7285
7286    struct WriteFileContributor;
7287
7288    impl ToolContributor for WriteFileContributor {
7289        fn id(&self) -> String {
7290            "test-write".to_string()
7291        }
7292
7293        fn contribute(&self, registry: &mut ToolRegistry) -> anyhow::Result<()> {
7294            registry.register(Arc::new(WriteFileTool))
7295        }
7296    }
7297
7298    struct WriteFileTool;
7299
7300    #[async_trait::async_trait]
7301    impl ToolExecutor for WriteFileTool {
7302        fn spec(&self) -> ToolSpec {
7303            ToolSpec {
7304                name: "write_file".to_string(),
7305                description: "Write a test file.".to_string(),
7306                parameters: serde_json::json!({
7307                    "type": "object",
7308                    "properties": {
7309                        "path": { "type": "string" },
7310                        "content": { "type": "string" }
7311                    },
7312                    "required": ["path", "content"],
7313                    "additionalProperties": false
7314                }),
7315            }
7316        }
7317
7318        async fn execute(
7319            &self,
7320            _ctx: ToolExecutionContext,
7321            call: ToolCall,
7322        ) -> anyhow::Result<ToolResult> {
7323            let path = call
7324                .arguments
7325                .get("path")
7326                .and_then(serde_json::Value::as_str)
7327                .unwrap_or("src/lib.rs");
7328            Ok(ToolResult {
7329                id: call.id,
7330                name: call.name,
7331                text: format!("wrote {path}"),
7332                data: serde_json::json!({ "path": path }),
7333                is_error: false,
7334            })
7335        }
7336    }
7337
7338    struct ProfileToolContributor;
7339
7340    impl ToolContributor for ProfileToolContributor {
7341        fn id(&self) -> String {
7342            "profile-tools".to_string()
7343        }
7344
7345        fn contribute(&self, registry: &mut ToolRegistry) -> anyhow::Result<()> {
7346            for name in ["apply_patch", "edit", "multi_edit", "write_file"] {
7347                registry.register(Arc::new(ProfileTool {
7348                    name: name.to_string(),
7349                }))?;
7350            }
7351            Ok(())
7352        }
7353    }
7354
7355    struct ProfileTool {
7356        name: String,
7357    }
7358
7359    #[async_trait::async_trait]
7360    impl ToolExecutor for ProfileTool {
7361        fn spec(&self) -> ToolSpec {
7362            ToolSpec {
7363                name: self.name.clone(),
7364                description: format!("{} test tool", self.name),
7365                parameters: serde_json::json!({
7366                    "type": "object",
7367                    "properties": {
7368                        "path": { "type": "string" },
7369                        "content": { "type": "string" }
7370                    },
7371                    "required": ["path", "content"],
7372                    "additionalProperties": false
7373                }),
7374            }
7375        }
7376
7377        async fn execute(
7378            &self,
7379            _ctx: ToolExecutionContext,
7380            call: ToolCall,
7381        ) -> anyhow::Result<ToolResult> {
7382            Ok(ToolResult {
7383                id: call.id,
7384                name: call.name,
7385                text: "ok".to_string(),
7386                data: serde_json::json!({}),
7387                is_error: false,
7388            })
7389        }
7390    }
7391
7392    fn test_model_profile(model: &str) -> ModelHarnessProfile {
7393        ModelHarnessProfile {
7394            model: model.to_string(),
7395            provider: roder_api::catalog::PROVIDER_OPENAI.to_string(),
7396            provider_family: ProviderFamily::OpenAi,
7397            edit_tool: Some(EDIT_TOOL_EDIT.to_string()),
7398            schema_policy: ModelSchemaPolicy::StandardRequiredFirst,
7399            instruction_overlay: ModelInstructionOverlay::IntuitiveContext,
7400            reasoning: ModelProfileReasoning {
7401                orientation: Some(REASONING_LOW.to_string()),
7402                execution: Some(REASONING_LOW.to_string()),
7403                verification: Some(REASONING_LOW.to_string()),
7404                recovery: Some(REASONING_LOW.to_string()),
7405            },
7406            parallel_tool_calls: Some(false),
7407            auto_compact_token_limit: Some(123_000),
7408        }
7409    }
7410
7411    async fn captured_profile_request(cfg: RuntimeConfig) -> AgentInferenceRequest {
7412        let captured = Arc::new(StdMutex::new(None));
7413        let mut builder = ExtensionRegistryBuilder::new();
7414        builder.inference_engine(Arc::new(CapturingEngine {
7415            request: captured.clone(),
7416        }));
7417        builder.tool_contributor(Arc::new(ProfileToolContributor));
7418        let runtime = Arc::new(Runtime::new(builder.build().unwrap(), cfg).unwrap());
7419        let mut rx = runtime.subscribe_events();
7420        let turn_id = runtime
7421            .start_turn(StartTurnRequest {
7422                thread_id: "thread-model-profile".to_string(),
7423                message: "use profile knobs".to_string(),
7424                images: Vec::new(),
7425                provider_override: None,
7426                model_override: None,
7427                reasoning_override: None,
7428                workspace: test_workspace(),
7429                instructions: InstructionBundle {
7430                    system: None,
7431                    developer: Some("base developer".to_string()),
7432                    developer_context: None,
7433                },
7434                developer_context: None,
7435                task_ledger_required: false,
7436                service_tier_override: None,
7437            })
7438            .await
7439            .unwrap();
7440
7441        tokio::time::timeout(std::time::Duration::from_secs(5), async {
7442            loop {
7443                let envelope = rx.recv().await.unwrap();
7444                if envelope.turn_id.as_deref() != Some(&turn_id) {
7445                    continue;
7446                }
7447                match envelope.event {
7448                    RoderEvent::TurnCompleted(_) => break,
7449                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7450                    _ => {}
7451                }
7452            }
7453        })
7454        .await
7455        .unwrap();
7456
7457        captured.lock().unwrap().clone().unwrap()
7458    }
7459
7460    struct ToolThenStopEngine {
7461        calls: StdMutex<u32>,
7462    }
7463
7464    #[async_trait::async_trait]
7465    impl InferenceEngine for ToolThenStopEngine {
7466        fn id(&self) -> String {
7467            roder_api::catalog::PROVIDER_MOCK.to_string()
7468        }
7469
7470        fn capabilities(&self) -> InferenceCapabilities {
7471            InferenceCapabilities::coding_agent_default()
7472        }
7473
7474        async fn list_models(
7475            &self,
7476            _ctx: InferenceProviderContext<'_>,
7477        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7478            Ok(roder_api::catalog::models_for_provider(
7479                roder_api::catalog::PROVIDER_MOCK,
7480                true,
7481            ))
7482        }
7483
7484        async fn stream_turn(
7485            &self,
7486            _ctx: InferenceTurnContext<'_>,
7487            _request: AgentInferenceRequest,
7488        ) -> anyhow::Result<InferenceEventStream> {
7489            let mut calls = self.calls.lock().unwrap();
7490            *calls += 1;
7491            let events = match *calls {
7492                1 => vec![
7493                    Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7494                        id: "write-1".to_string(),
7495                        name: "write_file".to_string(),
7496                        arguments: serde_json::json!({
7497                            "path": "src/lib.rs",
7498                            "content": "pub fn answer() -> u8 { 42 }\n"
7499                        })
7500                        .to_string(),
7501                    })),
7502                    Ok(InferenceEvent::Completed(CompletionMetadata {
7503                        stop_reason: Some("tool_use".to_string()),
7504                        provider_response_id: None,
7505                    })),
7506                ],
7507                _ => vec![
7508                    Ok(InferenceEvent::MessageDelta(MessageDelta {
7509                        text: "final".to_string(),
7510                        phase: None,
7511                    })),
7512                    Ok(InferenceEvent::Completed(CompletionMetadata {
7513                        stop_reason: Some("end_turn".to_string()),
7514                        provider_response_id: None,
7515                    })),
7516                ],
7517            };
7518            Ok(Box::pin(stream::iter(events)))
7519        }
7520    }
7521
7522    #[tokio::test]
7523    async fn turn_completed_reports_terminal_step_finish_reason() {
7524        let mut builder = ExtensionRegistryBuilder::new();
7525        builder.inference_engine(Arc::new(ToolThenStopEngine {
7526            calls: StdMutex::new(0),
7527        }));
7528        builder.tool_contributor(Arc::new(WriteFileContributor));
7529        let runtime = Arc::new(
7530            Runtime::new(
7531                builder.build().unwrap(),
7532                RuntimeConfig {
7533                    policy_mode: PolicyMode::Bypass,
7534                    agent_swarm_mode: false,
7535                    ultra_mode: false,
7536                    ..RuntimeConfig::default()
7537                },
7538            )
7539            .unwrap(),
7540        );
7541        let mut rx = runtime.subscribe_events();
7542        let turn_id = runtime
7543            .start_turn(StartTurnRequest {
7544                thread_id: "thread-finish-reason".to_string(),
7545                message: "write then finish".to_string(),
7546                images: Vec::new(),
7547                provider_override: None,
7548                model_override: None,
7549                reasoning_override: None,
7550                workspace: test_workspace(),
7551                instructions: InstructionBundle {
7552                    system: None,
7553                    developer: None,
7554                    developer_context: None,
7555                },
7556                developer_context: None,
7557                task_ledger_required: false,
7558                service_tier_override: None,
7559            })
7560            .await
7561            .unwrap();
7562
7563        let completed = tokio::time::timeout(std::time::Duration::from_secs(5), async {
7564            loop {
7565                let envelope = rx.recv().await.unwrap();
7566                if envelope.turn_id.as_deref() != Some(&turn_id) {
7567                    continue;
7568                }
7569                match envelope.event {
7570                    RoderEvent::TurnCompleted(event) => break event,
7571                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7572                    _ => {}
7573                }
7574            }
7575        })
7576        .await
7577        .unwrap();
7578
7579        // The mid-turn tool_use stop reason must not leak; the terminal
7580        // end_turn step decides the turn's finish reason.
7581        assert_eq!(completed.finish_reason.as_deref(), Some("stop"));
7582    }
7583
7584    #[tokio::test]
7585    async fn inference_router_selection_changes_request_model_and_records_event() {
7586        let default_requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7587        let routed_requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7588        let contexts = Arc::new(StdMutex::new(Vec::<InferenceRoutingContext>::new()));
7589        let selected = ModelSelection {
7590            provider: "routed-provider".to_string(),
7591            model: "routed-model".to_string(),
7592        };
7593        let default = ModelSelection {
7594            provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7595            model: "mock".to_string(),
7596        };
7597        let decision = InferenceRoutingDecision {
7598            reasoning: Some(ReasoningConfig {
7599                enabled: true,
7600                level: Some(REASONING_LOW.to_string()),
7601            }),
7602            confidence: Some(0.91),
7603            baseline: Some(default.clone()),
7604            matched_signals: vec![roder_api::inference_routing::InferenceRoutingSignal::new(
7605                "intent", "routine",
7606            )],
7607            ..InferenceRoutingDecision::selected("test-router", selected.clone(), "routine request")
7608        };
7609
7610        let mut builder = ExtensionRegistryBuilder::new();
7611        builder.inference_engine(Arc::new(RoutingCaptureEngine {
7612            id: roder_api::catalog::PROVIDER_MOCK,
7613            models: vec![routing_test_model("mock", &[REASONING_LOW])],
7614            requests: default_requests.clone(),
7615        }));
7616        builder.inference_engine(Arc::new(RoutingCaptureEngine {
7617            id: "routed-provider",
7618            models: vec![routing_test_model(
7619                "routed-model",
7620                &[REASONING_LOW, REASONING_MEDIUM],
7621            )],
7622            requests: routed_requests.clone(),
7623        }));
7624        builder.inference_router(Arc::new(StaticRouter {
7625            id: "test-router",
7626            decision,
7627            contexts: contexts.clone(),
7628        }));
7629        let thread_root =
7630            std::env::temp_dir().join(format!("roder-routing-auto-{}", uuid::Uuid::new_v4()));
7631        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
7632            base_path: thread_root.clone(),
7633        }));
7634        let runtime = Arc::new(
7635            Runtime::new(
7636                builder.build().unwrap(),
7637                RuntimeConfig {
7638                    default_provider: default.provider.clone(),
7639                    default_model: default.model.clone(),
7640                    ..RuntimeConfig::default()
7641                },
7642            )
7643            .unwrap(),
7644        );
7645        let thread_id = runtime
7646            .create_thread_with(CreateThreadRequest {
7647                title: Some("Routing auto".to_string()),
7648                workspace: test_workspace(),
7649                workspace_id: None,
7650                root_id: None,
7651                provider: Some(default.provider.clone()),
7652                model: Some(default.model.clone()),
7653                tool_allowlist: Vec::new(),
7654                service_tier: None,
7655                developer_instructions: None,
7656                external_tools: Vec::new(),
7657                selection_mode: Some(ModelSelectionMode::auto(
7658                    "test-router:coding",
7659                    "test-router",
7660                    "Auto: Coding",
7661                    default.clone(),
7662                    Some("coding".to_string()),
7663                    None,
7664                )),
7665                runner: None,
7666            })
7667            .await
7668            .unwrap()
7669            .thread_id;
7670        let mut rx = runtime.subscribe_events();
7671        let turn_id = runtime
7672            .start_turn(StartTurnRequest {
7673                thread_id: thread_id.clone(),
7674                message: "small cleanup".to_string(),
7675                images: Vec::new(),
7676                provider_override: None,
7677                model_override: None,
7678                reasoning_override: None,
7679                workspace: test_workspace(),
7680                instructions: InstructionBundle::default(),
7681                developer_context: None,
7682                task_ledger_required: false,
7683                service_tier_override: None,
7684            })
7685            .await
7686            .unwrap();
7687
7688        let mut routing_event = None;
7689        let mut inference_started = None;
7690        tokio::time::timeout(std::time::Duration::from_secs(5), async {
7691            loop {
7692                let envelope = rx.recv().await.unwrap();
7693                if envelope.turn_id.as_deref() != Some(&turn_id) {
7694                    continue;
7695                }
7696                match envelope.event {
7697                    RoderEvent::InferenceRoutingDecision(event) => {
7698                        routing_event = Some(event);
7699                    }
7700                    RoderEvent::InferenceStarted(event) => {
7701                        inference_started = Some(event);
7702                    }
7703                    RoderEvent::TurnCompleted(_) => break,
7704                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7705                    _ => {}
7706                }
7707            }
7708        })
7709        .await
7710        .unwrap();
7711
7712        assert!(default_requests.lock().unwrap().is_empty());
7713        let routed_requests = routed_requests.lock().unwrap();
7714        assert_eq!(routed_requests.len(), 1);
7715        assert_eq!(routed_requests[0].model, selected);
7716        assert_eq!(
7717            routed_requests[0].reasoning.level.as_deref(),
7718            Some(REASONING_LOW)
7719        );
7720        assert_eq!(
7721            routed_requests[0].metadata["inferenceRouting"]["outcome"],
7722            "selected"
7723        );
7724
7725        let routing_event = routing_event.expect("routing decision event");
7726        assert_eq!(routing_event.default_selection, default);
7727        assert_eq!(routing_event.selected_selection, selected);
7728        assert_eq!(
7729            routing_event.decision.outcome,
7730            InferenceRoutingOutcome::Selected
7731        );
7732        assert_eq!(
7733            inference_started.expect("inference started event").model,
7734            selected
7735        );
7736
7737        let contexts = contexts.lock().unwrap();
7738        assert_eq!(contexts.len(), 1);
7739        assert_eq!(contexts[0].default_selection, default);
7740        assert_eq!(contexts[0].candidates.len(), 2);
7741        assert!(
7742            contexts[0]
7743                .signals
7744                .iter()
7745                .any(|signal| signal.key == "profile" && signal.value == "coding")
7746        );
7747        let _ = std::fs::remove_dir_all(thread_root);
7748    }
7749
7750    #[tokio::test]
7751    async fn inference_router_is_bypassed_for_explicit_selection() {
7752        let requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7753        let contexts = Arc::new(StdMutex::new(Vec::<InferenceRoutingContext>::new()));
7754        let selected = ModelSelection {
7755            provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7756            model: "mock".to_string(),
7757        };
7758
7759        let mut builder = ExtensionRegistryBuilder::new();
7760        builder.inference_engine(Arc::new(RoutingCaptureEngine {
7761            id: roder_api::catalog::PROVIDER_MOCK,
7762            models: vec![routing_test_model("mock", &[REASONING_LOW])],
7763            requests: requests.clone(),
7764        }));
7765        builder.inference_router(Arc::new(StaticRouter {
7766            id: "test-router",
7767            decision: InferenceRoutingDecision::selected(
7768                "test-router",
7769                ModelSelection {
7770                    provider: "missing".to_string(),
7771                    model: "missing".to_string(),
7772                },
7773                "would route if called",
7774            ),
7775            contexts: contexts.clone(),
7776        }));
7777        let thread_root =
7778            std::env::temp_dir().join(format!("roder-routing-explicit-{}", uuid::Uuid::new_v4()));
7779        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
7780            base_path: thread_root.clone(),
7781        }));
7782        let runtime = Arc::new(
7783            Runtime::new(
7784                builder.build().unwrap(),
7785                RuntimeConfig {
7786                    default_provider: selected.provider.clone(),
7787                    default_model: selected.model.clone(),
7788                    inference_router: RuntimeInferenceRouterConfig {
7789                        enabled: true,
7790                        router_id: Some("test-router".to_string()),
7791                    },
7792                    ..RuntimeConfig::default()
7793                },
7794            )
7795            .unwrap(),
7796        );
7797        let thread_id = runtime
7798            .create_thread_with(CreateThreadRequest {
7799                title: Some("Routing explicit".to_string()),
7800                workspace: test_workspace(),
7801                workspace_id: None,
7802                root_id: None,
7803                provider: Some(selected.provider.clone()),
7804                model: Some(selected.model.clone()),
7805                tool_allowlist: Vec::new(),
7806                service_tier: None,
7807                developer_instructions: None,
7808                external_tools: Vec::new(),
7809                selection_mode: Some(ModelSelectionMode::auto(
7810                    "test-router:default",
7811                    "test-router",
7812                    "Auto",
7813                    selected.clone(),
7814                    None,
7815                    None,
7816                )),
7817                runner: None,
7818            })
7819            .await
7820            .unwrap()
7821            .thread_id;
7822        let mut rx = runtime.subscribe_events();
7823        let turn_id = runtime
7824            .start_turn(StartTurnRequest {
7825                thread_id,
7826                message: "use explicit selection".to_string(),
7827                images: Vec::new(),
7828                provider_override: Some(selected.provider.clone()),
7829                model_override: Some(selected.model.clone()),
7830                reasoning_override: None,
7831                workspace: test_workspace(),
7832                instructions: InstructionBundle::default(),
7833                developer_context: None,
7834                task_ledger_required: false,
7835                service_tier_override: None,
7836            })
7837            .await
7838            .unwrap();
7839
7840        let mut saw_routing_event = false;
7841        tokio::time::timeout(std::time::Duration::from_secs(5), async {
7842            loop {
7843                let envelope = rx.recv().await.unwrap();
7844                if envelope.turn_id.as_deref() != Some(&turn_id) {
7845                    continue;
7846                }
7847                match envelope.event {
7848                    RoderEvent::InferenceRoutingDecision(_) => {
7849                        saw_routing_event = true;
7850                    }
7851                    RoderEvent::TurnCompleted(_) => break,
7852                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7853                    _ => {}
7854                }
7855            }
7856        })
7857        .await
7858        .unwrap();
7859
7860        assert!(!saw_routing_event);
7861        assert!(contexts.lock().unwrap().is_empty());
7862        let requests = requests.lock().unwrap();
7863        assert_eq!(requests.len(), 1);
7864        assert_eq!(requests[0].model, selected);
7865        let _ = std::fs::remove_dir_all(thread_root);
7866    }
7867
7868    #[tokio::test]
7869    async fn inference_router_is_bypassed_for_manual_selection_mode() {
7870        let requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7871        let contexts = Arc::new(StdMutex::new(Vec::<InferenceRoutingContext>::new()));
7872        let selected = ModelSelection {
7873            provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7874            model: "mock".to_string(),
7875        };
7876
7877        let mut builder = ExtensionRegistryBuilder::new();
7878        builder.inference_engine(Arc::new(RoutingCaptureEngine {
7879            id: roder_api::catalog::PROVIDER_MOCK,
7880            models: vec![routing_test_model("mock", &[REASONING_LOW])],
7881            requests: requests.clone(),
7882        }));
7883        builder.inference_router(Arc::new(StaticRouter {
7884            id: "test-router",
7885            decision: InferenceRoutingDecision::selected(
7886                "test-router",
7887                ModelSelection {
7888                    provider: "missing".to_string(),
7889                    model: "missing".to_string(),
7890                },
7891                "would route if called",
7892            ),
7893            contexts: contexts.clone(),
7894        }));
7895        let thread_root =
7896            std::env::temp_dir().join(format!("roder-routing-manual-{}", uuid::Uuid::new_v4()));
7897        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
7898            base_path: thread_root.clone(),
7899        }));
7900        let runtime = Arc::new(
7901            Runtime::new(
7902                builder.build().unwrap(),
7903                RuntimeConfig {
7904                    default_provider: selected.provider.clone(),
7905                    default_model: selected.model.clone(),
7906                    inference_router: RuntimeInferenceRouterConfig {
7907                        enabled: true,
7908                        router_id: Some("test-router".to_string()),
7909                    },
7910                    ..RuntimeConfig::default()
7911                },
7912            )
7913            .unwrap(),
7914        );
7915        let thread_id = runtime
7916            .create_thread_with(CreateThreadRequest {
7917                title: Some("Routing manual".to_string()),
7918                workspace: test_workspace(),
7919                workspace_id: None,
7920                root_id: None,
7921                provider: Some(selected.provider.clone()),
7922                model: Some(selected.model.clone()),
7923                tool_allowlist: Vec::new(),
7924                service_tier: None,
7925                developer_instructions: None,
7926                external_tools: Vec::new(),
7927                selection_mode: Some(ModelSelectionMode::manual(
7928                    selected.provider.clone(),
7929                    selected.model.clone(),
7930                    None,
7931                )),
7932                runner: None,
7933            })
7934            .await
7935            .unwrap()
7936            .thread_id;
7937        let mut rx = runtime.subscribe_events();
7938        let turn_id = runtime
7939            .start_turn(StartTurnRequest {
7940                thread_id,
7941                message: "use selected manual model".to_string(),
7942                images: Vec::new(),
7943                provider_override: None,
7944                model_override: None,
7945                reasoning_override: None,
7946                workspace: test_workspace(),
7947                instructions: InstructionBundle::default(),
7948                developer_context: None,
7949                task_ledger_required: false,
7950                service_tier_override: None,
7951            })
7952            .await
7953            .unwrap();
7954
7955        let mut saw_routing_event = false;
7956        tokio::time::timeout(std::time::Duration::from_secs(5), async {
7957            loop {
7958                let envelope = rx.recv().await.unwrap();
7959                if envelope.turn_id.as_deref() != Some(&turn_id) {
7960                    continue;
7961                }
7962                match envelope.event {
7963                    RoderEvent::InferenceRoutingDecision(_) => {
7964                        saw_routing_event = true;
7965                    }
7966                    RoderEvent::TurnCompleted(_) => break,
7967                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7968                    _ => {}
7969                }
7970            }
7971        })
7972        .await
7973        .unwrap();
7974
7975        assert!(!saw_routing_event);
7976        assert!(contexts.lock().unwrap().is_empty());
7977        let requests = requests.lock().unwrap();
7978        assert_eq!(requests.len(), 1);
7979        assert_eq!(requests[0].model, selected);
7980        let _ = std::fs::remove_dir_all(thread_root);
7981    }
7982
7983    #[test]
7984    fn enabled_inference_router_requires_registered_router() {
7985        let mut builder = ExtensionRegistryBuilder::new();
7986        builder.inference_engine(Arc::new(FakeInferenceEngine));
7987
7988        let err = match Runtime::new(
7989            builder.build().unwrap(),
7990            RuntimeConfig {
7991                inference_router: RuntimeInferenceRouterConfig {
7992                    enabled: true,
7993                    router_id: Some("missing-router".to_string()),
7994                },
7995                ..RuntimeConfig::default()
7996            },
7997        ) {
7998            Ok(_) => panic!("runtime should reject unknown inference router"),
7999            Err(err) => err,
8000        };
8001
8002        assert!(
8003            err.to_string()
8004                .contains("inference router \"missing-router\" is not registered")
8005        );
8006    }
8007
8008    #[tokio::test]
8009    async fn model_profile_routes_request_knobs_to_next_inference() {
8010        let request = captured_profile_request(RuntimeConfig {
8011            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8012            default_model: "gpt-5.5".to_string(),
8013            model_profiles: std::collections::HashMap::from([(
8014                "gpt-5.5".to_string(),
8015                test_model_profile("gpt-5.5"),
8016            )]),
8017            ..RuntimeConfig::default()
8018        })
8019        .await;
8020
8021        let tool_names = request
8022            .tools
8023            .iter()
8024            .map(|tool| tool.name.as_str())
8025            .collect::<Vec<_>>();
8026        assert!(tool_names.contains(&"apply_patch"));
8027        assert!(tool_names.contains(&"edit"));
8028        assert!(tool_names.contains(&"multi_edit"));
8029        assert!(tool_names.contains(&"write_file"));
8030        assert_eq!(request.reasoning.level.as_deref(), Some(REASONING_LOW));
8031        assert_eq!(request.runtime.parallel_tool_calls, Some(false));
8032        assert_eq!(request.runtime.auto_compact_token_limit, Some(123_000));
8033        assert!(
8034            request
8035                .instructions
8036                .developer
8037                .as_deref()
8038                .unwrap_or_default()
8039                .contains("Use the provided context as the current working set")
8040        );
8041        assert_eq!(
8042            request
8043                .metadata
8044                .pointer("/modelProfile/schemaPolicy")
8045                .and_then(serde_json::Value::as_str),
8046            Some("standard_required_first")
8047        );
8048    }
8049
8050    #[tokio::test]
8051    async fn ultra_reasoning_reaches_transport_state_and_enables_proactive_delegation() {
8052        let request = captured_profile_request(RuntimeConfig {
8053            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8054            default_model: "gpt-5.6-sol".to_string(),
8055            reasoning: Some(REASONING_ULTRA.to_string()),
8056            ..RuntimeConfig::default()
8057        })
8058        .await;
8059
8060        assert_eq!(request.reasoning.level.as_deref(), Some(REASONING_ULTRA));
8061        let developer = request
8062            .instructions
8063            .developer
8064            .as_deref()
8065            .expect("ultra developer instructions");
8066        assert!(developer.contains("Proactive multi-agent delegation is active"));
8067    }
8068
8069    #[tokio::test]
8070    async fn lower_sol_effort_requires_explicit_multi_agent_request() {
8071        let request = captured_profile_request(RuntimeConfig {
8072            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8073            default_model: "gpt-5.6-sol".to_string(),
8074            reasoning: Some(roder_api::catalog::REASONING_MEDIUM.to_string()),
8075            ..RuntimeConfig::default()
8076        })
8077        .await;
8078
8079        let developer = request
8080            .instructions
8081            .developer
8082            .as_deref()
8083            .expect("sol developer instructions");
8084        assert!(developer.contains("Do not spawn sub-agents unless"));
8085        assert!(!developer.contains("Proactive multi-agent delegation is active"));
8086    }
8087
8088    #[tokio::test]
8089    async fn luna_does_not_receive_codex_v2_multi_agent_policy() {
8090        let request = captured_profile_request(RuntimeConfig {
8091            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8092            default_model: "gpt-5.6-luna".to_string(),
8093            reasoning: Some(roder_api::catalog::REASONING_MAX.to_string()),
8094            ..RuntimeConfig::default()
8095        })
8096        .await;
8097
8098        let developer = request
8099            .instructions
8100            .developer
8101            .as_deref()
8102            .unwrap_or_default();
8103        assert!(!developer.contains("Do not spawn sub-agents unless"));
8104        assert!(!developer.contains("Proactive multi-agent delegation is active"));
8105    }
8106
8107    #[tokio::test]
8108    async fn ultra_mode_enables_proactive_multi_agent_for_any_model() {
8109        // Luna has no Ultra effort, but ultra mode still injects proactive policy.
8110        let request = captured_profile_request(RuntimeConfig {
8111            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8112            default_model: "gpt-5.6-luna".to_string(),
8113            reasoning: Some(roder_api::catalog::REASONING_MAX.to_string()),
8114            ultra_mode: true,
8115            ..RuntimeConfig::default()
8116        })
8117        .await;
8118
8119        let developer = request
8120            .instructions
8121            .developer
8122            .as_deref()
8123            .expect("ultra mode developer instructions");
8124        assert!(developer.contains("Proactive multi-agent delegation is active"));
8125        assert!(developer.contains("spawn_agent"));
8126    }
8127
8128    #[tokio::test]
8129    async fn ultra_mode_overrides_explicit_request_only_on_sol() {
8130        // Medium Sol effort is normally explicit-request-only; ultra mode flips
8131        // it to proactive without requiring Ultra reasoning effort.
8132        let request = captured_profile_request(RuntimeConfig {
8133            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8134            default_model: "gpt-5.6-sol".to_string(),
8135            reasoning: Some(roder_api::catalog::REASONING_MEDIUM.to_string()),
8136            ultra_mode: true,
8137            ..RuntimeConfig::default()
8138        })
8139        .await;
8140
8141        let developer = request
8142            .instructions
8143            .developer
8144            .as_deref()
8145            .expect("ultra mode sol developer instructions");
8146        assert!(developer.contains("Proactive multi-agent delegation is active"));
8147        assert!(!developer.contains("Do not spawn sub-agents unless"));
8148    }
8149
8150    #[tokio::test]
8151    async fn turn_developer_context_reaches_inference_and_does_not_persist() {
8152        let captured = Arc::new(StdMutex::new(None));
8153        let mut builder = ExtensionRegistryBuilder::new();
8154        builder.inference_engine(Arc::new(CapturingEngine {
8155            request: captured.clone(),
8156        }));
8157        let runtime = Arc::new(
8158            Runtime::new(
8159                builder.build().unwrap(),
8160                RuntimeConfig {
8161                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8162                    default_model: "gpt-5.5".to_string(),
8163                    ..RuntimeConfig::default()
8164                },
8165            )
8166            .unwrap(),
8167        );
8168
8169        async fn run_turn(runtime: &Arc<Runtime>, developer_context: Option<String>) {
8170            let mut rx = runtime.subscribe_events();
8171            let turn_id = runtime
8172                .start_turn(StartTurnRequest {
8173                    thread_id: "thread-turn-context".to_string(),
8174                    message: "hello".to_string(),
8175                    images: Vec::new(),
8176                    provider_override: None,
8177                    model_override: None,
8178                    reasoning_override: None,
8179                    workspace: test_workspace(),
8180                    instructions: InstructionBundle::default(),
8181                    developer_context,
8182                    task_ledger_required: false,
8183                    service_tier_override: None,
8184                })
8185                .await
8186                .unwrap();
8187            tokio::time::timeout(std::time::Duration::from_secs(5), async {
8188                loop {
8189                    let envelope = rx.recv().await.unwrap();
8190                    if envelope.turn_id.as_deref() != Some(&turn_id) {
8191                        continue;
8192                    }
8193                    match envelope.event {
8194                        RoderEvent::TurnCompleted(_) => break,
8195                        RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8196                        _ => {}
8197                    }
8198                }
8199            })
8200            .await
8201            .unwrap();
8202        }
8203
8204        run_turn(
8205            &runtime,
8206            Some("Connected accounts: example-service.".to_string()),
8207        )
8208        .await;
8209        let request = captured.lock().unwrap().clone().unwrap();
8210        assert_eq!(
8211            request.instructions.developer_context.as_deref(),
8212            Some("Connected accounts: example-service.")
8213        );
8214
8215        // The context is per-turn only: the next turn on the same thread
8216        // without a developerContext must not see the previous one.
8217        run_turn(&runtime, None).await;
8218        let request = captured.lock().unwrap().clone().unwrap();
8219        assert_eq!(request.instructions.developer_context, None);
8220    }
8221
8222    #[tokio::test]
8223    async fn tool_search_overrides_route_to_next_inference_request() {
8224        let request = captured_profile_request(RuntimeConfig {
8225            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8226            default_model: "gpt-5.4".to_string(),
8227            tool_search: ToolSearchConfig {
8228                mode: roder_api::inference::ToolSearchMode::Auto,
8229                max_catalog_items: Some(100),
8230                ..ToolSearchConfig::default()
8231            },
8232            provider_tool_search: std::collections::HashMap::from([(
8233                roder_api::catalog::PROVIDER_MOCK.to_string(),
8234                roder_api::inference::ToolSearchConfigOverlay {
8235                    include_skills: Some(false),
8236                    provider_variant: Some(roder_api::inference::ToolSearchProviderVariant::Regex),
8237                    ..Default::default()
8238                },
8239            )]),
8240            model_tool_search: std::collections::HashMap::from([(
8241                "gpt-5.4".to_string(),
8242                roder_api::inference::ToolSearchConfigOverlay {
8243                    mode: Some(roder_api::inference::ToolSearchMode::ProviderNative),
8244                    max_catalog_items: Some(25),
8245                    provider_variant: Some(roder_api::inference::ToolSearchProviderVariant::Bm25),
8246                    ..Default::default()
8247                },
8248            )]),
8249            ..RuntimeConfig::default()
8250        })
8251        .await;
8252
8253        assert_eq!(
8254            request.runtime.tool_search.mode,
8255            roder_api::inference::ToolSearchMode::ProviderNative
8256        );
8257        assert_eq!(request.runtime.tool_search.max_catalog_items, Some(25));
8258        assert_eq!(
8259            request.runtime.tool_search.provider_variant,
8260            roder_api::inference::ToolSearchProviderVariant::Bm25
8261        );
8262    }
8263
8264    #[tokio::test]
8265    async fn context_entrypoint_hints_use_turn_workspace() {
8266        let process_workspace = runtime_test_workspace("entrypoint-process");
8267        let thread_workspace = runtime_test_workspace("entrypoint-thread");
8268        std::fs::create_dir_all(process_workspace.join("src")).unwrap();
8269        std::fs::create_dir_all(thread_workspace.join("src")).unwrap();
8270        std::fs::write(
8271            process_workspace.join("src/sidebar-thread-groups.ts"),
8272            "export const desktopLeak = true;\n",
8273        )
8274        .unwrap();
8275        std::fs::write(
8276            thread_workspace.join("src/voice-plan-feedback.ts"),
8277            "export const voicePlanFeedback = true;\n",
8278        )
8279        .unwrap();
8280
8281        let captured = Arc::new(StdMutex::new(None));
8282        let mut builder = ExtensionRegistryBuilder::new();
8283        builder.inference_engine(Arc::new(CapturingEngine {
8284            request: captured.clone(),
8285        }));
8286        builder.context_planner(Arc::new(roder_context::EntrypointContextPlanner::new(
8287            process_workspace.clone(),
8288        )));
8289        let runtime = Arc::new(
8290            Runtime::new(
8291                builder.build().unwrap(),
8292                RuntimeConfig {
8293                    workspace: Some(process_workspace.display().to_string()),
8294                    ..RuntimeConfig::default()
8295                },
8296            )
8297            .unwrap(),
8298        );
8299        let mut rx = runtime.subscribe_events();
8300
8301        let turn_id = runtime
8302            .start_turn(StartTurnRequest {
8303                thread_id: "thread-workspace-entrypoint".to_string(),
8304                message: "investigate voice plan feedback".to_string(),
8305                images: Vec::new(),
8306                provider_override: None,
8307                model_override: None,
8308                reasoning_override: None,
8309                workspace: thread_workspace.display().to_string(),
8310                instructions: crate::instructions::default_instructions(),
8311                developer_context: None,
8312                task_ledger_required: false,
8313                service_tier_override: None,
8314            })
8315            .await
8316            .unwrap();
8317
8318        tokio::time::timeout(std::time::Duration::from_secs(5), async {
8319            loop {
8320                let envelope = rx.recv().await.unwrap();
8321                if envelope.turn_id.as_deref() != Some(&turn_id) {
8322                    continue;
8323                }
8324                match envelope.event {
8325                    RoderEvent::TurnCompleted(_) => break,
8326                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8327                    _ => {}
8328                }
8329            }
8330        })
8331        .await
8332        .unwrap();
8333
8334        let request = captured.lock().unwrap().clone().expect("captured request");
8335        let transcript_text = request
8336            .transcript
8337            .iter()
8338            .map(|item| match item {
8339                TranscriptItem::UserMessage(message) => message.text.as_str(),
8340                _ => "",
8341            })
8342            .collect::<Vec<_>>()
8343            .join("\n");
8344        assert!(transcript_text.contains("src/voice-plan-feedback.ts"));
8345        assert!(!transcript_text.contains("src/sidebar-thread-groups.ts"));
8346
8347        let _ = std::fs::remove_dir_all(process_workspace);
8348        let _ = std::fs::remove_dir_all(thread_workspace);
8349    }
8350
8351    fn runtime_test_workspace(name: &str) -> std::path::PathBuf {
8352        let path =
8353            std::env::temp_dir().join(format!("roder-runtime-{name}-{}", uuid::Uuid::new_v4()));
8354        let _ = std::fs::remove_dir_all(&path);
8355        std::fs::create_dir_all(&path).unwrap();
8356        path
8357    }
8358
8359    #[tokio::test]
8360    async fn model_profile_user_model_knobs_override_profile_defaults() {
8361        let request = captured_profile_request(RuntimeConfig {
8362            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8363            default_model: "gpt-5.5".to_string(),
8364            reasoning: Some(REASONING_HIGH.to_string()),
8365            auto_compact_token_limit: Some(999),
8366            model_edit_tools: std::collections::HashMap::from([(
8367                "gpt-5.5".to_string(),
8368                EDIT_TOOL_PATCH.to_string(),
8369            )]),
8370            model_parallel_tool_calls: std::collections::HashMap::from([(
8371                "gpt-5.5".to_string(),
8372                true,
8373            )]),
8374            model_profiles: std::collections::HashMap::from([(
8375                "gpt-5.5".to_string(),
8376                test_model_profile("gpt-5.5"),
8377            )]),
8378            ..RuntimeConfig::default()
8379        })
8380        .await;
8381
8382        let tool_names = request
8383            .tools
8384            .iter()
8385            .map(|tool| tool.name.as_str())
8386            .collect::<Vec<_>>();
8387        assert!(tool_names.contains(&"apply_patch"));
8388        assert!(!tool_names.contains(&"edit"));
8389        assert_eq!(request.reasoning.level.as_deref(), Some(REASONING_HIGH));
8390        assert_eq!(request.runtime.parallel_tool_calls, Some(true));
8391        assert_eq!(request.runtime.auto_compact_token_limit, Some(999));
8392    }
8393
8394    #[tokio::test]
8395    async fn runtime_tool_allowlist_filters_advertised_tools() {
8396        let request = captured_profile_request(RuntimeConfig {
8397            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8398            default_model: "gpt-5.5".to_string(),
8399            tool_allowlist: vec!["edit".to_string()],
8400            model_profiles: std::collections::HashMap::from([(
8401                "gpt-5.5".to_string(),
8402                test_model_profile("gpt-5.5"),
8403            )]),
8404            ..RuntimeConfig::default()
8405        })
8406        .await;
8407
8408        let tool_names = request
8409            .tools
8410            .iter()
8411            .map(|tool| tool.name.as_str())
8412            .collect::<Vec<_>>();
8413        assert_eq!(tool_names, vec!["edit"]);
8414    }
8415
8416    /// Builds a store-backed runtime turn for a thread created with the given per-thread
8417    /// overrides and returns the captured inference request.
8418    async fn captured_thread_override_request(
8419        runtime: &Arc<Runtime>,
8420        requests: &Arc<StdMutex<Vec<AgentInferenceRequest>>>,
8421        tool_allowlist: Vec<String>,
8422        developer_instructions: Option<String>,
8423        external_tools: Vec<ToolSpec>,
8424    ) -> AgentInferenceRequest {
8425        let thread_id = runtime
8426            .create_thread_with(CreateThreadRequest {
8427                title: Some("Thread overrides".to_string()),
8428                workspace: test_workspace(),
8429                workspace_id: None,
8430                root_id: None,
8431                provider: None,
8432                model: None,
8433                selection_mode: None,
8434                tool_allowlist,
8435                service_tier: None,
8436                developer_instructions,
8437                external_tools,
8438                runner: None,
8439            })
8440            .await
8441            .unwrap()
8442            .thread_id;
8443        let mut rx = runtime.subscribe_events();
8444        let turn_id = runtime
8445            .start_turn(StartTurnRequest {
8446                thread_id,
8447                message: "hello".to_string(),
8448                images: Vec::new(),
8449                provider_override: None,
8450                model_override: None,
8451                reasoning_override: None,
8452                workspace: test_workspace(),
8453                instructions: crate::default_instructions(),
8454                developer_context: None,
8455                task_ledger_required: false,
8456                service_tier_override: None,
8457            })
8458            .await
8459            .unwrap();
8460        tokio::time::timeout(std::time::Duration::from_secs(5), async {
8461            loop {
8462                let envelope = rx.recv().await.unwrap();
8463                if envelope.turn_id.as_deref() != Some(&turn_id) {
8464                    continue;
8465                }
8466                match envelope.event {
8467                    RoderEvent::TurnCompleted(_) => break,
8468                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8469                    _ => {}
8470                }
8471            }
8472        })
8473        .await
8474        .unwrap();
8475        requests.lock().unwrap().pop().expect("captured request")
8476    }
8477
8478    #[tokio::test]
8479    async fn thread_tool_allowlist_filters_only_that_thread() {
8480        let requests = Arc::new(StdMutex::new(Vec::new()));
8481        let thread_root =
8482            std::env::temp_dir().join(format!("roder-thread-allowlist-{}", uuid::Uuid::new_v4()));
8483        let mut builder = ExtensionRegistryBuilder::new();
8484        builder.inference_engine(Arc::new(SwitchCaptureEngine {
8485            requests: requests.clone(),
8486        }));
8487        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8488            base_path: thread_root.clone(),
8489        }));
8490        builder.tool_contributor(Arc::new(ProfileToolContributor));
8491        let runtime = Arc::new(
8492            Runtime::new(
8493                builder.build().unwrap(),
8494                RuntimeConfig {
8495                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8496                    default_model: "gpt-5.5".to_string(),
8497                    model_profiles: std::collections::HashMap::from([(
8498                        "gpt-5.5".to_string(),
8499                        test_model_profile("gpt-5.5"),
8500                    )]),
8501                    ..RuntimeConfig::default()
8502                },
8503            )
8504            .unwrap(),
8505        );
8506
8507        let allowlisted = captured_thread_override_request(
8508            &runtime,
8509            &requests,
8510            vec!["edit".to_string()],
8511            None,
8512            Vec::new(),
8513        )
8514        .await;
8515        let unrestricted =
8516            captured_thread_override_request(&runtime, &requests, Vec::new(), None, Vec::new())
8517                .await;
8518
8519        let allowlisted_names = allowlisted
8520            .tools
8521            .iter()
8522            .map(|tool| tool.name.as_str())
8523            .collect::<Vec<_>>();
8524        assert_eq!(allowlisted_names, vec!["edit"]);
8525        let unrestricted_names = unrestricted
8526            .tools
8527            .iter()
8528            .map(|tool| tool.name.as_str())
8529            .collect::<Vec<_>>();
8530        assert!(unrestricted_names.contains(&"edit"));
8531        assert!(unrestricted_names.len() > 1);
8532
8533        let _ = std::fs::remove_dir_all(thread_root);
8534    }
8535
8536    /// Builds a store-backed runtime whose `RuntimeConfig.tool_allowlist` is `["edit"]`.
8537    fn runtime_with_edit_allowlist(
8538        requests: &Arc<StdMutex<Vec<AgentInferenceRequest>>>,
8539        thread_root: &std::path::Path,
8540    ) -> Arc<Runtime> {
8541        let mut builder = ExtensionRegistryBuilder::new();
8542        builder.inference_engine(Arc::new(SwitchCaptureEngine {
8543            requests: requests.clone(),
8544        }));
8545        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8546            base_path: thread_root.to_path_buf(),
8547        }));
8548        builder.tool_contributor(Arc::new(ProfileToolContributor));
8549        Arc::new(
8550            Runtime::new(
8551                builder.build().unwrap(),
8552                RuntimeConfig {
8553                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8554                    default_model: "gpt-5.5".to_string(),
8555                    tool_allowlist: vec!["edit".to_string()],
8556                    model_profiles: std::collections::HashMap::from([(
8557                        "gpt-5.5".to_string(),
8558                        test_model_profile("gpt-5.5"),
8559                    )]),
8560                    ..RuntimeConfig::default()
8561                },
8562            )
8563            .unwrap(),
8564        )
8565    }
8566
8567    #[tokio::test]
8568    async fn runtime_and_thread_allowlists_intersect() {
8569        let requests = Arc::new(StdMutex::new(Vec::new()));
8570        let thread_root = std::env::temp_dir().join(format!(
8571            "roder-allowlist-intersect-{}",
8572            uuid::Uuid::new_v4()
8573        ));
8574        let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8575
8576        let request = captured_thread_override_request(
8577            &runtime,
8578            &requests,
8579            vec!["edit".to_string(), "write_file".to_string()],
8580            None,
8581            Vec::new(),
8582        )
8583        .await;
8584
8585        // The thread allowlist must not re-enable tools the runtime allowlist bans.
8586        let names = request
8587            .tools
8588            .iter()
8589            .map(|tool| tool.name.as_str())
8590            .collect::<Vec<_>>();
8591        assert_eq!(names, vec!["edit"]);
8592
8593        let _ = std::fs::remove_dir_all(thread_root);
8594    }
8595
8596    #[tokio::test]
8597    async fn route_tool_call_denies_tools_outside_allowlists() {
8598        let requests = Arc::new(StdMutex::new(Vec::new()));
8599        let thread_root =
8600            std::env::temp_dir().join(format!("roder-allowlist-dispatch-{}", uuid::Uuid::new_v4()));
8601        let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8602        let thread_id = runtime
8603            .create_thread_with(CreateThreadRequest {
8604                title: Some("Dispatch allowlist".to_string()),
8605                workspace: test_workspace(),
8606                workspace_id: None,
8607                root_id: None,
8608                provider: None,
8609                model: None,
8610                selection_mode: None,
8611                tool_allowlist: vec!["edit".to_string(), "write_file".to_string()],
8612                service_tier: None,
8613                developer_instructions: None,
8614                external_tools: Vec::new(),
8615                runner: None,
8616            })
8617            .await
8618            .unwrap()
8619            .thread_id;
8620
8621        // write_file is registered and on the thread allowlist but banned by the runtime allowlist.
8622        let result = runtime
8623            .route_tool_call(
8624                &thread_id,
8625                &"turn-allowlist-dispatch".to_string(),
8626                roder_api::inference::ToolCallCompleted {
8627                    id: "call-1".to_string(),
8628                    name: "write_file".to_string(),
8629                    arguments: r#"{"path":"a.txt","content":"hi"}"#.to_string(),
8630                },
8631                None,
8632                None,
8633                None,
8634            )
8635            .await
8636            .unwrap();
8637
8638        assert!(result.is_error);
8639        assert!(
8640            result
8641                .result
8642                .contains("not permitted by the tool allowlist"),
8643            "unexpected result: {}",
8644            result.result
8645        );
8646
8647        let _ = std::fs::remove_dir_all(thread_root);
8648    }
8649
8650    #[tokio::test]
8651    async fn route_tool_calls_denies_agent_swarm_mixed_with_other_tools() {
8652        let requests = Arc::new(StdMutex::new(Vec::new()));
8653        let thread_root =
8654            std::env::temp_dir().join(format!("roder-swarm-exclusive-{}", uuid::Uuid::new_v4()));
8655        let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8656
8657        // A response that mixes agent_swarm with another tool is denied wholesale:
8658        // both calls get an error result with retry guidance, and no tool runs.
8659        let results = runtime
8660            .route_tool_calls(
8661                &"thread-swarm".to_string(),
8662                &"turn-swarm".to_string(),
8663                vec![
8664                    roder_api::inference::ToolCallCompleted {
8665                        id: "swarm-1".to_string(),
8666                        name: "agent_swarm".to_string(),
8667                        arguments: "{}".to_string(),
8668                    },
8669                    roder_api::inference::ToolCallCompleted {
8670                        id: "read-1".to_string(),
8671                        name: "read_file".to_string(),
8672                        arguments: "{}".to_string(),
8673                    },
8674                ],
8675                true,
8676                None,
8677                None,
8678            )
8679            .await
8680            .unwrap();
8681
8682        assert_eq!(results.len(), 2, "every tool_call_id must get a response");
8683        assert_eq!(results[0].id, "swarm-1");
8684        assert_eq!(results[1].id, "read-1");
8685        for result in &results {
8686            assert!(result.is_error);
8687            assert!(
8688                result.result.contains("only tool call"),
8689                "unexpected result: {}",
8690                result.result
8691            );
8692        }
8693
8694        let _ = std::fs::remove_dir_all(thread_root);
8695    }
8696
8697    #[tokio::test]
8698    async fn route_tool_calls_emits_agent_swarm_started_event() {
8699        let requests = Arc::new(StdMutex::new(Vec::new()));
8700        let thread_root =
8701            std::env::temp_dir().join(format!("roder-swarm-started-{}", uuid::Uuid::new_v4()));
8702        let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8703        let mut events = runtime.subscribe_events();
8704
8705        // A single agent_swarm call (the valid shape) emits AgentSwarmStarted on
8706        // the event bus before dispatch, with the child count parsed from args.
8707        let _ = runtime
8708            .route_tool_calls(
8709                &"thread-swarm".to_string(),
8710                &"turn-swarm".to_string(),
8711                vec![roder_api::inference::ToolCallCompleted {
8712                    id: "swarm-1".to_string(),
8713                    name: "agent_swarm".to_string(),
8714                    arguments: r#"{"description":"x","prompt_template":"Read {{item}}","items":["a.rs","b.rs"]}"#
8715                        .to_string(),
8716                }],
8717                true,
8718                None,
8719                None,
8720            )
8721            .await
8722            .unwrap();
8723
8724        let mut started_child_count = None;
8725        for _ in 0..16 {
8726            let envelope = tokio::time::timeout(std::time::Duration::from_secs(2), events.recv())
8727                .await
8728                .unwrap()
8729                .unwrap();
8730            if let RoderEvent::AgentSwarmStarted(event) = envelope.event {
8731                started_child_count = Some(event.child_count);
8732                assert_eq!(event.tool_id, "swarm-1");
8733                break;
8734            }
8735        }
8736        assert_eq!(started_child_count, Some(2));
8737
8738        let _ = std::fs::remove_dir_all(thread_root);
8739    }
8740
8741    #[test]
8742    fn agent_swarm_child_count_sums_items_and_resumes() {
8743        assert_eq!(
8744            agent_swarm_child_count(r#"{"description":"x","items":["a","b","c"]}"#),
8745            3
8746        );
8747        assert_eq!(
8748            agent_swarm_child_count(
8749                r#"{"description":"x","items":["a"],"resume_agent_ids":{"id1":"continue"}}"#
8750            ),
8751            2
8752        );
8753        // Malformed input is lenient.
8754        assert_eq!(agent_swarm_child_count("not json"), 0);
8755    }
8756
8757    #[test]
8758    fn parse_swarm_counts_reads_summary_with_omitted_buckets() {
8759        let text = "<agent_swarm_result>\n<summary>completed: 2, failed: 1</summary>\n</agent_swarm_result>";
8760        assert_eq!(parse_swarm_counts(text), Some((2, 1, 0)));
8761        let text = "<agent_swarm_result>\n<summary>completed: 0</summary>\n</agent_swarm_result>";
8762        assert_eq!(parse_swarm_counts(text), Some((0, 0, 0)));
8763        // Not a swarm result.
8764        assert_eq!(parse_swarm_counts("just text"), None);
8765    }
8766
8767    /// Signals when inference starts, then waits for `proceed` and fails the stream.
8768    struct SignalledFailureEngine {
8769        started: tokio::sync::mpsc::UnboundedSender<()>,
8770        proceed: Arc<tokio::sync::Notify>,
8771    }
8772
8773    #[async_trait::async_trait]
8774    impl InferenceEngine for SignalledFailureEngine {
8775        fn id(&self) -> String {
8776            roder_api::catalog::PROVIDER_MOCK.to_string()
8777        }
8778
8779        fn capabilities(&self) -> InferenceCapabilities {
8780            InferenceCapabilities::coding_agent_default()
8781        }
8782
8783        async fn list_models(
8784            &self,
8785            _ctx: InferenceProviderContext<'_>,
8786        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
8787            Ok(roder_api::catalog::models_for_provider(
8788                roder_api::catalog::PROVIDER_MOCK,
8789                true,
8790            ))
8791        }
8792
8793        async fn stream_turn(
8794            &self,
8795            _ctx: InferenceTurnContext<'_>,
8796            _request: AgentInferenceRequest,
8797        ) -> anyhow::Result<InferenceEventStream> {
8798            let _ = self.started.send(());
8799            self.proceed.notified().await;
8800            anyhow::bail!("engine failed mid-turn")
8801        }
8802    }
8803
8804    #[tokio::test]
8805    async fn failed_turn_sweeps_pending_external_tool_calls() {
8806        let (started_tx, mut started_rx) = tokio::sync::mpsc::unbounded_channel();
8807        let proceed = Arc::new(tokio::sync::Notify::new());
8808        let mut builder = ExtensionRegistryBuilder::new();
8809        builder.inference_engine(Arc::new(SignalledFailureEngine {
8810            started: started_tx,
8811            proceed: proceed.clone(),
8812        }));
8813        let runtime =
8814            Arc::new(Runtime::new(builder.build().unwrap(), RuntimeConfig::default()).unwrap());
8815        let mut rx = runtime.subscribe_events();
8816        let turn_id = runtime
8817            .start_turn(StartTurnRequest {
8818                thread_id: "thread-sweep".to_string(),
8819                message: "go".to_string(),
8820                images: Vec::new(),
8821                provider_override: None,
8822                model_override: None,
8823                reasoning_override: None,
8824                workspace: test_workspace(),
8825                instructions: InstructionBundle {
8826                    system: None,
8827                    developer: None,
8828                    developer_context: None,
8829                },
8830                developer_context: None,
8831                task_ledger_required: false,
8832                service_tier_override: None,
8833            })
8834            .await
8835            .unwrap();
8836        tokio::time::timeout(std::time::Duration::from_secs(5), started_rx.recv())
8837            .await
8838            .unwrap()
8839            .unwrap();
8840
8841        let (tx, _pending_rx) = oneshot::channel();
8842        runtime.pending_external_tool_calls.lock().await.insert(
8843            "exttool-sweep-test".to_string(),
8844            PendingExternalToolCall {
8845                thread_id: "thread-sweep".to_string(),
8846                turn_id: turn_id.clone(),
8847                tool_id: "call-1".to_string(),
8848                tool_name: "acme_lookup".to_string(),
8849                tx,
8850            },
8851        );
8852        proceed.notify_one();
8853
8854        let outcome = tokio::time::timeout(std::time::Duration::from_secs(5), async {
8855            loop {
8856                let envelope = rx.recv().await.unwrap();
8857                if let RoderEvent::ExternalToolCallResolved(event) = envelope.event
8858                    && event.request_id == "exttool-sweep-test"
8859                {
8860                    break event.outcome;
8861                }
8862            }
8863        })
8864        .await
8865        .expect("turn failure must resolve pending external tool calls");
8866        assert_eq!(outcome, ExternalToolCallOutcome::Cancelled);
8867        assert!(runtime.pending_external_tool_calls.lock().await.is_empty());
8868    }
8869
8870    #[tokio::test]
8871    async fn thread_developer_instructions_layer_under_harness_prompt() {
8872        let requests = Arc::new(StdMutex::new(Vec::new()));
8873        let thread_root = std::env::temp_dir().join(format!(
8874            "roder-thread-instructions-{}",
8875            uuid::Uuid::new_v4()
8876        ));
8877        let mut builder = ExtensionRegistryBuilder::new();
8878        builder.inference_engine(Arc::new(SwitchCaptureEngine {
8879            requests: requests.clone(),
8880        }));
8881        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8882            base_path: thread_root.clone(),
8883        }));
8884        builder.tool_contributor(Arc::new(ProfileToolContributor));
8885        let runtime =
8886            Arc::new(Runtime::new(builder.build().unwrap(), RuntimeConfig::default()).unwrap());
8887
8888        let request = captured_thread_override_request(
8889            &runtime,
8890            &requests,
8891            Vec::new(),
8892            Some("You are embedded in a host app.".to_string()),
8893            Vec::new(),
8894        )
8895        .await;
8896
8897        let system = request.instructions.system.expect("system instructions");
8898        assert!(system.starts_with("You are Roder"));
8899        let developer = request
8900            .instructions
8901            .developer
8902            .expect("developer instructions");
8903        assert!(developer.starts_with("You are embedded in a host app."));
8904
8905        let plain =
8906            captured_thread_override_request(&runtime, &requests, Vec::new(), None, Vec::new())
8907                .await;
8908        assert_eq!(plain.instructions.developer, None);
8909
8910        let _ = std::fs::remove_dir_all(thread_root);
8911    }
8912
8913    #[tokio::test]
8914    async fn thread_external_tools_are_advertised_and_shadow_builtins() {
8915        let requests = Arc::new(StdMutex::new(Vec::new()));
8916        let thread_root = std::env::temp_dir().join(format!(
8917            "roder-thread-external-tools-{}",
8918            uuid::Uuid::new_v4()
8919        ));
8920        let mut builder = ExtensionRegistryBuilder::new();
8921        builder.inference_engine(Arc::new(SwitchCaptureEngine {
8922            requests: requests.clone(),
8923        }));
8924        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8925            base_path: thread_root.clone(),
8926        }));
8927        builder.tool_contributor(Arc::new(ProfileToolContributor));
8928        let runtime =
8929            Arc::new(Runtime::new(builder.build().unwrap(), RuntimeConfig::default()).unwrap());
8930
8931        let external_tools = vec![
8932            ToolSpec {
8933                name: "acme_lookup".to_string(),
8934                description: "Look up Acme workspace state.".to_string(),
8935                parameters: serde_json::json!({
8936                    "type": "object",
8937                    "properties": { "query": { "type": "string" } },
8938                    "required": ["query"]
8939                }),
8940            },
8941            ToolSpec {
8942                name: "edit".to_string(),
8943                description: "Host-managed edit.".to_string(),
8944                parameters: serde_json::json!({ "type": "object" }),
8945            },
8946        ];
8947        let request =
8948            captured_thread_override_request(&runtime, &requests, Vec::new(), None, external_tools)
8949                .await;
8950
8951        let acme = request
8952            .tools
8953            .iter()
8954            .find(|tool| tool.name == "acme_lookup")
8955            .expect("external tool advertised");
8956        assert_eq!(acme.description, "Look up Acme workspace state.");
8957        assert_eq!(acme.parameters["required"][0], "query");
8958        let edits = request
8959            .tools
8960            .iter()
8961            .filter(|tool| tool.name == "edit")
8962            .collect::<Vec<_>>();
8963        assert_eq!(edits.len(), 1, "external edit shadows the builtin");
8964        assert_eq!(edits[0].description, "Host-managed edit.");
8965
8966        let plain =
8967            captured_thread_override_request(&runtime, &requests, Vec::new(), None, Vec::new())
8968                .await;
8969        assert!(plain.tools.iter().all(|tool| tool.name != "acme_lookup"));
8970        let plain_edit = plain
8971            .tools
8972            .iter()
8973            .find(|tool| tool.name == "edit")
8974            .expect("builtin edit advertised on plain thread");
8975        assert_eq!(plain_edit.description, "edit test tool");
8976
8977        let _ = std::fs::remove_dir_all(thread_root);
8978    }
8979
8980    #[tokio::test]
8981    async fn model_selection_reads_and_updates_metadata_without_loading_history() {
8982        let thread_root = tempfile::tempdir().unwrap();
8983        let mut builder = ExtensionRegistryBuilder::new();
8984        builder.inference_engine(Arc::new(SwitchCaptureEngine {
8985            requests: Arc::new(StdMutex::new(Vec::new())),
8986        }));
8987        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8988            base_path: thread_root.path().to_path_buf(),
8989        }));
8990        let runtime = Runtime::new(builder.build().unwrap(), RuntimeConfig::default()).unwrap();
8991        let thread = runtime
8992            .create_thread(Some("Metadata-only model switch".to_string()))
8993            .await
8994            .unwrap();
8995        let dir = thread_root.path().join(&thread.thread_id);
8996        // Invalid history makes any accidental full-snapshot read fail.
8997        for file in ["events.jsonl", "item_events.jsonl", "extension_state.jsonl"] {
8998            tokio::fs::write(dir.join(file), b"history must not be read\n")
8999                .await
9000                .unwrap();
9001        }
9002        let selection = ModelSelectionMode::manual(
9003            roder_api::catalog::PROVIDER_MOCK.to_string(),
9004            "gpt-5.5".to_string(),
9005            Some("medium".to_string()),
9006        );
9007
9008        runtime
9009            .set_thread_selection_mode(&thread.thread_id, selection.clone())
9010            .await
9011            .unwrap();
9012        assert_eq!(
9013            runtime
9014                .selection_mode_for_thread(&thread.thread_id)
9015                .await
9016                .unwrap(),
9017            Some(selection.clone())
9018        );
9019        let metadata = runtime
9020            .load_thread_metadata(&thread.thread_id)
9021            .await
9022            .unwrap()
9023            .unwrap();
9024        assert_eq!(metadata.selection_mode, Some(selection));
9025        assert_eq!(metadata.provider, thread.provider);
9026        assert_eq!(metadata.model.as_deref(), Some("gpt-5.5"));
9027        assert_eq!(metadata.title, thread.title);
9028        assert_eq!(metadata.workspace, thread.workspace);
9029        for file in ["events.jsonl", "item_events.jsonl", "extension_state.jsonl"] {
9030            assert_eq!(
9031                tokio::fs::read(dir.join(file)).await.unwrap(),
9032                b"history must not be read\n"
9033            );
9034        }
9035        assert!(
9036            runtime
9037                .set_thread_selection_mode(
9038                    &"missing-thread".to_string(),
9039                    ModelSelectionMode::manual("mock", "mock", None),
9040                )
9041                .await
9042                .unwrap_err()
9043                .to_string()
9044                .contains("thread not found")
9045        );
9046    }
9047
9048    #[tokio::test]
9049    async fn model_switch_injects_summary_and_records_profile_segments() {
9050        let requests = Arc::new(StdMutex::new(Vec::new()));
9051        let thread_root = std::env::temp_dir().join(format!(
9052            "roder-model-switch-thread-{}",
9053            uuid::Uuid::new_v4()
9054        ));
9055        let mut builder = ExtensionRegistryBuilder::new();
9056        builder.inference_engine(Arc::new(SwitchCaptureEngine {
9057            requests: requests.clone(),
9058        }));
9059        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
9060            base_path: thread_root.clone(),
9061        }));
9062        builder.tool_contributor(Arc::new(ProfileToolContributor));
9063        let mut claude_profile = test_model_profile("claude-haiku-4-5-20251001");
9064        claude_profile.provider_family = ProviderFamily::Anthropic;
9065        claude_profile.edit_tool = Some(EDIT_TOOL_EDIT.to_string());
9066        let runtime = Arc::new(
9067            Runtime::new(
9068                builder.build().unwrap(),
9069                RuntimeConfig {
9070                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9071                    default_model: "gpt-5.5".to_string(),
9072                    model_profiles: std::collections::HashMap::from([
9073                        ("gpt-5.5".to_string(), test_model_profile("gpt-5.5")),
9074                        ("claude-haiku-4-5-20251001".to_string(), claude_profile),
9075                    ]),
9076                    ..RuntimeConfig::default()
9077                },
9078            )
9079            .unwrap(),
9080        );
9081        let thread_id = runtime
9082            .create_thread_with(CreateThreadRequest {
9083                title: Some("Model switch".to_string()),
9084                workspace: test_workspace(),
9085                workspace_id: None,
9086                root_id: None,
9087                provider: None,
9088                model: None,
9089                selection_mode: None,
9090                tool_allowlist: Vec::new(),
9091                service_tier: None,
9092                developer_instructions: None,
9093                external_tools: Vec::new(),
9094                runner: None,
9095            })
9096            .await
9097            .unwrap()
9098            .thread_id;
9099        let mut rx = runtime.subscribe_events();
9100        for (message, model_override) in [
9101            ("first turn", None),
9102            ("second turn", Some("claude-haiku-4-5-20251001".to_string())),
9103        ] {
9104            let turn_id = runtime
9105                .start_turn(StartTurnRequest {
9106                    thread_id: thread_id.clone(),
9107                    message: message.to_string(),
9108                    images: Vec::new(),
9109                    provider_override: None,
9110                    model_override,
9111                    reasoning_override: None,
9112                    workspace: test_workspace(),
9113                    instructions: InstructionBundle::default(),
9114                    developer_context: None,
9115                    task_ledger_required: false,
9116                    service_tier_override: None,
9117                })
9118                .await
9119                .unwrap();
9120            tokio::time::timeout(std::time::Duration::from_secs(5), async {
9121                loop {
9122                    let envelope = rx.recv().await.unwrap();
9123                    if envelope.turn_id.as_deref() != Some(&turn_id) {
9124                        continue;
9125                    }
9126                    match envelope.event {
9127                        RoderEvent::TurnCompleted(_) => break,
9128                        RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9129                        _ => {}
9130                    }
9131                }
9132            })
9133            .await
9134            .unwrap();
9135        }
9136
9137        let captured = requests.lock().unwrap().clone();
9138        assert_eq!(captured.len(), 2);
9139        assert!(captured[1].transcript.iter().any(|item| {
9140            matches!(
9141                item,
9142                TranscriptItem::UserMessage(message)
9143                    if message.text.starts_with(MODEL_SWITCH_SUMMARY_PREFIX)
9144                        && message.text.contains("previous profile mock/gpt-5.5")
9145                        && message.text.contains("Current profile mock/claude-haiku-4-5-20251001")
9146                        && message.text.contains("Available tools now:")
9147            )
9148        }));
9149
9150        let snapshot = runtime
9151            .thread_store
9152            .as_ref()
9153            .unwrap()
9154            .load_thread(&thread_id)
9155            .await
9156            .unwrap()
9157            .unwrap();
9158        let trace_segments = snapshot
9159            .turns
9160            .iter()
9161            .flat_map(|turn| &turn.items)
9162            .filter(|item| {
9163                matches!(
9164                    item,
9165                    TranscriptItem::ProviderMetadata(value)
9166                        if value.get("kind").and_then(serde_json::Value::as_str)
9167                            == Some(MODEL_PROFILE_TRACE_KIND)
9168                            && value.get("segment").and_then(serde_json::Value::as_str)
9169                                == Some("assistant")
9170                )
9171            })
9172            .count();
9173        assert!(trace_segments >= 2);
9174        let _ = std::fs::remove_dir_all(thread_root);
9175    }
9176
9177    struct CountingTaskTool {
9178        calls: Arc<StdMutex<u32>>,
9179    }
9180
9181    #[async_trait::async_trait]
9182    impl ToolExecutor for CountingTaskTool {
9183        fn spec(&self) -> ToolSpec {
9184            ToolSpec {
9185                name: "task".to_string(),
9186                description: "Dispatch a test subagent.".to_string(),
9187                parameters: serde_json::json!({
9188                    "type": "object",
9189                    "properties": {
9190                        "description": { "type": "string" },
9191                        "prompt": { "type": "string" },
9192                        "parent_deadline_seconds": { "type": "integer" }
9193                    },
9194                    "required": ["description", "prompt"],
9195                    "additionalProperties": false
9196                }),
9197            }
9198        }
9199
9200        async fn execute(
9201            &self,
9202            _ctx: ToolExecutionContext,
9203            call: ToolCall,
9204        ) -> anyhow::Result<ToolResult> {
9205            *self.calls.lock().unwrap() += 1;
9206            Ok(ToolResult {
9207                id: call.id,
9208                name: call.name,
9209                text: "started child".to_string(),
9210                data: serde_json::json!({}),
9211                is_error: false,
9212            })
9213        }
9214    }
9215
9216    #[tokio::test]
9217    async fn runtime_profile_reaches_inference_request_and_turn_metadata() {
9218        let captured = Arc::new(StdMutex::new(None));
9219        let mut builder = ExtensionRegistryBuilder::new();
9220        builder.inference_engine(Arc::new(CapturingEngine {
9221            request: captured.clone(),
9222        }));
9223        let runtime = Arc::new(
9224            Runtime::new(
9225                builder.build().unwrap(),
9226                RuntimeConfig {
9227                    runtime_profile: RuntimeProfile::NonInteractive,
9228                    ..RuntimeConfig::default()
9229                },
9230            )
9231            .unwrap(),
9232        );
9233        let mut rx = runtime.subscribe_events();
9234        let turn_id = runtime
9235            .start_turn(StartTurnRequest {
9236                thread_id: "thread-profile".to_string(),
9237                message: "work unattended".to_string(),
9238                images: Vec::new(),
9239                provider_override: None,
9240                model_override: None,
9241                reasoning_override: None,
9242                workspace: test_workspace(),
9243                instructions: InstructionBundle {
9244                    system: None,
9245                    developer: Some("base developer".to_string()),
9246                    developer_context: None,
9247                },
9248                developer_context: None,
9249                task_ledger_required: false,
9250                service_tier_override: None,
9251            })
9252            .await
9253            .unwrap();
9254
9255        let mut observed_profile = None;
9256        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9257            loop {
9258                let envelope = rx.recv().await.unwrap();
9259                if envelope.turn_id.as_deref() != Some(&turn_id) {
9260                    continue;
9261                }
9262                match envelope.event {
9263                    RoderEvent::TurnStarted(event) => {
9264                        observed_profile = Some(event.runtime_profile);
9265                    }
9266                    RoderEvent::TurnCompleted(_) => break,
9267                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9268                    _ => {}
9269                }
9270            }
9271        })
9272        .await
9273        .unwrap();
9274
9275        assert_eq!(observed_profile, Some(RuntimeProfile::NonInteractive));
9276        let request = captured.lock().unwrap().clone().unwrap();
9277        assert_eq!(request.runtime.profile, RuntimeProfile::NonInteractive);
9278        let developer = request.instructions.developer.unwrap();
9279        assert!(developer.contains("base developer"));
9280        assert!(developer.contains("non-interactive profile"));
9281    }
9282
9283    #[tokio::test]
9284    async fn global_policy_mode_changes_do_not_create_runtime_thread_directory() {
9285        let workspace = runtime_test_workspace("global-policy-mode");
9286        let thread_root = workspace.join("threads");
9287        let mut builder = ExtensionRegistryBuilder::new();
9288        builder.inference_engine(Arc::new(FakeInferenceEngine));
9289        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
9290            base_path: thread_root.clone(),
9291        }));
9292        let runtime = Runtime::new(
9293            builder.build().unwrap(),
9294            RuntimeConfig {
9295                workspace: Some(workspace.display().to_string()),
9296                ..Default::default()
9297            },
9298        )
9299        .unwrap();
9300
9301        runtime
9302            .set_policy_mode(PolicyMode::AcceptAll, Some("test".to_string()))
9303            .await
9304            .unwrap();
9305
9306        assert!(!thread_root.join("runtime").exists());
9307        let _ = std::fs::remove_dir_all(workspace);
9308    }
9309
9310    #[tokio::test]
9311    async fn task_ledger_enforcement_injects_eval_reminder_before_work() {
9312        let captured = Arc::new(StdMutex::new(None));
9313        let mut builder = ExtensionRegistryBuilder::new();
9314        builder.inference_engine(Arc::new(CapturingEngine {
9315            request: captured.clone(),
9316        }));
9317        builder.tool_contributor(Arc::new(
9318            roder_ext_task_ledger::TaskLedgerToolContributor::default(),
9319        ));
9320        let runtime = Arc::new(
9321            Runtime::new(
9322                builder.build().unwrap(),
9323                RuntimeConfig {
9324                    runtime_profile: RuntimeProfile::Eval,
9325                    policy_mode: PolicyMode::Bypass,
9326                    agent_swarm_mode: false,
9327                    ultra_mode: false,
9328                    ..RuntimeConfig::default()
9329                },
9330            )
9331            .unwrap(),
9332        );
9333        let mut rx = runtime.subscribe_events();
9334        let turn_id = runtime
9335            .start_turn(StartTurnRequest {
9336                thread_id: "thread-ledger".to_string(),
9337                message: "decomposed work".to_string(),
9338                images: Vec::new(),
9339                provider_override: None,
9340                model_override: None,
9341                reasoning_override: None,
9342                workspace: test_workspace(),
9343                instructions: InstructionBundle::default(),
9344                developer_context: None,
9345                task_ledger_required: true,
9346                service_tier_override: None,
9347            })
9348            .await
9349            .unwrap();
9350
9351        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9352            loop {
9353                let envelope = rx.recv().await.unwrap();
9354                if envelope.turn_id.as_deref() == Some(&turn_id)
9355                    && matches!(envelope.event, RoderEvent::TurnCompleted(_))
9356                {
9357                    break;
9358                }
9359            }
9360        })
9361        .await
9362        .unwrap();
9363
9364        let request = captured.lock().unwrap().clone().unwrap();
9365        let developer = request.instructions.developer.unwrap();
9366        assert!(developer.contains("Task Ledger Required"));
9367        assert!(developer.contains("task_ledger.update"));
9368        let tool_names: Vec<_> = request
9369            .tools
9370            .iter()
9371            .map(|tool| tool.name.as_str())
9372            .collect();
9373        assert!(
9374            tool_names.contains(&TASK_LEDGER_TOOL_NAME),
9375            "tool names: {tool_names:?}"
9376        );
9377        assert_eq!(
9378            request.tool_choice,
9379            ToolChoice::Specific(TASK_LEDGER_TOOL_NAME.to_string())
9380        );
9381        assert_eq!(request.tools.len(), 1);
9382        assert_eq!(request.tools[0].name, TASK_LEDGER_TOOL_NAME);
9383    }
9384
9385    #[tokio::test]
9386    async fn eval_task_ledger_blocks_final_answer_until_open_items_are_completed() {
9387        let requests = Arc::new(StdMutex::new(Vec::new()));
9388        let mut builder = ExtensionRegistryBuilder::new();
9389        builder.inference_engine(Arc::new(TaskLedgerCompletionGateEngine {
9390            calls: StdMutex::new(0),
9391            requests: requests.clone(),
9392        }));
9393        builder.tool_contributor(Arc::new(
9394            roder_ext_task_ledger::TaskLedgerToolContributor::default(),
9395        ));
9396        let runtime = Arc::new(
9397            Runtime::new(
9398                builder.build().unwrap(),
9399                RuntimeConfig {
9400                    runtime_profile: RuntimeProfile::Eval,
9401                    policy_mode: PolicyMode::Bypass,
9402                    agent_swarm_mode: false,
9403                    ultra_mode: false,
9404                    ..RuntimeConfig::default()
9405                },
9406            )
9407            .unwrap(),
9408        );
9409        let mut rx = runtime.subscribe_events();
9410        let turn_id = runtime
9411            .start_turn(StartTurnRequest {
9412                thread_id: "thread-ledger-completion".to_string(),
9413                message: "write the answer file".to_string(),
9414                images: Vec::new(),
9415                provider_override: None,
9416                model_override: None,
9417                reasoning_override: None,
9418                workspace: test_workspace(),
9419                instructions: InstructionBundle::default(),
9420                developer_context: None,
9421                task_ledger_required: true,
9422                service_tier_override: None,
9423            })
9424            .await
9425            .unwrap();
9426
9427        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9428            loop {
9429                let envelope = rx.recv().await.unwrap();
9430                if envelope.turn_id.as_deref() != Some(&turn_id) {
9431                    continue;
9432                }
9433                match envelope.event {
9434                    RoderEvent::TurnCompleted(_) => break,
9435                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9436                    _ => {}
9437                }
9438            }
9439        })
9440        .await
9441        .unwrap();
9442
9443        let requests = requests.lock().unwrap().clone();
9444        assert_eq!(requests.len(), 4);
9445        assert!(requests[2].transcript.iter().any(|item| {
9446            matches!(
9447                item,
9448                TranscriptItem::UserMessage(message)
9449                    if message.text.contains("Task Ledger Completion Required")
9450                        && message.text.contains("Write /app/result.txt")
9451            )
9452        }));
9453        assert!(requests[3].transcript.iter().any(|item| {
9454            matches!(
9455                item,
9456                TranscriptItem::ToolResult(result)
9457                    if result.name.as_deref() == Some(TASK_LEDGER_TOOL_NAME)
9458                        && result.result.contains("Task ledger: 2/2 completed")
9459            )
9460        }));
9461    }
9462
9463    #[tokio::test]
9464    async fn eval_task_ledger_checkpoint_requests_scoreable_file_before_final_reserve() {
9465        let requests = Arc::new(StdMutex::new(Vec::new()));
9466        let mut builder = ExtensionRegistryBuilder::new();
9467        builder.inference_engine(Arc::new(TaskLedgerCompletionGateEngine {
9468            calls: StdMutex::new(0),
9469            requests: requests.clone(),
9470        }));
9471        builder.tool_contributor(Arc::new(
9472            roder_ext_task_ledger::TaskLedgerToolContributor::default(),
9473        ));
9474        let runtime = Arc::new(
9475            Runtime::new(
9476                builder.build().unwrap(),
9477                RuntimeConfig {
9478                    runtime_profile: RuntimeProfile::Eval,
9479                    policy_mode: PolicyMode::Bypass,
9480                    agent_swarm_mode: false,
9481                    ultra_mode: false,
9482                    turn_deadline_seconds: Some(120),
9483                    ..RuntimeConfig::default()
9484                },
9485            )
9486            .unwrap(),
9487        );
9488        let mut rx = runtime.subscribe_events();
9489        let turn_id = runtime
9490            .start_turn(StartTurnRequest {
9491                thread_id: "thread-ledger-checkpoint".to_string(),
9492                message: "write the answer file".to_string(),
9493                images: Vec::new(),
9494                provider_override: None,
9495                model_override: None,
9496                reasoning_override: None,
9497                workspace: test_workspace(),
9498                instructions: InstructionBundle::default(),
9499                developer_context: None,
9500                task_ledger_required: true,
9501                service_tier_override: None,
9502            })
9503            .await
9504            .unwrap();
9505
9506        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9507            loop {
9508                let envelope = rx.recv().await.unwrap();
9509                if envelope.turn_id.as_deref() != Some(&turn_id) {
9510                    continue;
9511                }
9512                match envelope.event {
9513                    RoderEvent::TurnCompleted(_) => break,
9514                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9515                    _ => {}
9516                }
9517            }
9518        })
9519        .await
9520        .unwrap();
9521
9522        let requests = requests.lock().unwrap().clone();
9523        assert!(requests.len() >= 2);
9524        assert!(requests[1].transcript.iter().any(|item| {
9525            matches!(
9526                item,
9527                TranscriptItem::UserMessage(message)
9528                    if message.text.contains("Scoreable Output Checkpoint")
9529                        && message.text.contains("ensure the required output file(s) exist")
9530                        && message.text.contains("Write /app/result.txt")
9531            )
9532        }));
9533    }
9534
9535    #[test]
9536    fn deadline_task_ledger_prompt_preserves_scoreable_work_instruction() {
9537        let prompt = task_ledger_deadline_completion_prompt(
9538            12,
9539            30,
9540            "Task Ledger Completion Required: write /app/result.txt, then call task_ledger.update",
9541        );
9542
9543        assert!(prompt.contains("12 seconds remain"));
9544        assert!(prompt.contains("create or update the required scoreable output files"));
9545        assert!(prompt.contains("write /app/result.txt"));
9546        assert!(prompt.contains(TASK_LEDGER_TOOL_NAME));
9547    }
9548
9549    #[test]
9550    fn scoreable_checkpoint_prompt_preserves_provisional_file_instruction() {
9551        let prompt = task_ledger_scoreable_checkpoint_prompt(
9552            120,
9553            "Task Ledger Completion Required: write /app/result.txt, then call task_ledger.update",
9554        );
9555
9556        assert!(prompt.contains("120 seconds remain"));
9557        assert!(prompt.contains("best evidence-backed answer"));
9558        assert!(prompt.contains("even if provisional"));
9559        assert!(prompt.contains("preserve that candidate"));
9560        assert!(prompt.contains("partial-coverage"));
9561        assert!(prompt.contains("write /app/result.txt"));
9562        assert!(prompt.contains(TASK_LEDGER_TOOL_NAME));
9563    }
9564
9565    #[test]
9566    fn open_task_ledger_moves_inference_timeout_to_scoreable_checkpoint() {
9567        let deadline = Some(OffsetDateTime::now_utc() + Duration::seconds(870));
9568        let transcript = vec![TranscriptItem::ToolResult(ToolResultRecord {
9569            id: "ledger-open".to_string(),
9570            name: Some(TASK_LEDGER_TOOL_NAME.to_string()),
9571            result: "Task ledger: 0/1 completed\n- pending: Write /app/result.txt [write]"
9572                .to_string(),
9573            display_payload: None,
9574            is_error: false,
9575        })];
9576
9577        let (_, action) = inference_timeout_deadline(
9578            deadline,
9579            RuntimeProfile::Eval,
9580            true,
9581            30,
9582            false,
9583            0,
9584            &transcript,
9585        )
9586        .unwrap();
9587
9588        assert_eq!(action, InferenceTimeoutAction::ScoreableCheckpoint);
9589    }
9590
9591    #[tokio::test]
9592    async fn verification_gate_forces_eval_code_changes_through_review() {
9593        let mut builder = ExtensionRegistryBuilder::new();
9594        builder.inference_engine(Arc::new(VerificationGateEngine {
9595            calls: StdMutex::new(0),
9596        }));
9597        builder.tool_contributor(Arc::new(WriteFileContributor));
9598        builder.tool_contributor(Arc::new(
9599            roder_ext_verification::VerificationToolContributor,
9600        ));
9601        let runtime = Arc::new(
9602            Runtime::new(
9603                builder.build().unwrap(),
9604                RuntimeConfig {
9605                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9606                    default_model: "mock".to_string(),
9607                    runtime_profile: RuntimeProfile::Eval,
9608                    policy_mode: PolicyMode::Bypass,
9609                    agent_swarm_mode: false,
9610                    ultra_mode: false,
9611                    ..RuntimeConfig::default()
9612                },
9613            )
9614            .unwrap(),
9615        );
9616        let mut rx = runtime.subscribe_events();
9617        let turn_id = runtime
9618            .start_turn(StartTurnRequest {
9619                thread_id: "thread-verification".to_string(),
9620                message: "write code".to_string(),
9621                images: Vec::new(),
9622                provider_override: None,
9623                model_override: None,
9624                reasoning_override: None,
9625                workspace: test_workspace(),
9626                instructions: InstructionBundle::default(),
9627                developer_context: None,
9628                task_ledger_required: false,
9629                service_tier_override: None,
9630            })
9631            .await
9632            .unwrap();
9633
9634        let mut saw_required = false;
9635        let mut saw_completed = false;
9636        let mut final_text = String::new();
9637        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9638            loop {
9639                let envelope = rx.recv().await.unwrap();
9640                if envelope.turn_id.as_deref() != Some(&turn_id) {
9641                    continue;
9642                }
9643                match envelope.event {
9644                    RoderEvent::VerificationRequired(event) => {
9645                        saw_required = true;
9646                        assert_eq!(event.changed_files, vec!["src/lib.rs"]);
9647                    }
9648                    RoderEvent::VerificationCompleted(event) => {
9649                        saw_completed = true;
9650                        assert!(event.passed);
9651                    }
9652                    RoderEvent::InferenceEventReceived(event) => {
9653                        if let InferenceEvent::MessageDelta(delta) = event.event {
9654                            final_text.push_str(&delta.text);
9655                        }
9656                    }
9657                    RoderEvent::TurnCompleted(_) => break,
9658                    _ => {}
9659                }
9660            }
9661        })
9662        .await
9663        .unwrap();
9664
9665        assert!(saw_required);
9666        assert!(saw_completed);
9667        assert!(final_text.contains("verified final"));
9668    }
9669
9670    #[tokio::test]
9671    async fn speed_policy_changes_reasoning_across_eval_model_calls_without_model_switch() {
9672        let requests = Arc::new(StdMutex::new(Vec::new()));
9673        let mut builder = ExtensionRegistryBuilder::new();
9674        builder.inference_engine(Arc::new(SpeedPolicyEngine {
9675            calls: StdMutex::new(0),
9676            requests: requests.clone(),
9677        }));
9678        builder.tool_contributor(Arc::new(WriteFileContributor));
9679        builder.tool_contributor(Arc::new(
9680            roder_ext_verification::VerificationToolContributor,
9681        ));
9682        let runtime = Arc::new(
9683            Runtime::new(
9684                builder.build().unwrap(),
9685                RuntimeConfig {
9686                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9687                    default_model: "gpt-5.5".to_string(),
9688                    runtime_profile: RuntimeProfile::Eval,
9689                    policy_mode: PolicyMode::Bypass,
9690                    agent_swarm_mode: false,
9691                    ultra_mode: false,
9692                    ..RuntimeConfig::default()
9693                },
9694            )
9695            .unwrap(),
9696        );
9697        let mut rx = runtime.subscribe_events();
9698        let turn_id = runtime
9699            .start_turn(StartTurnRequest {
9700                thread_id: "thread-speed-policy".to_string(),
9701                message: "write code".to_string(),
9702                images: Vec::new(),
9703                provider_override: None,
9704                model_override: None,
9705                reasoning_override: None,
9706                workspace: test_workspace(),
9707                instructions: InstructionBundle::default(),
9708                developer_context: None,
9709                task_ledger_required: false,
9710                service_tier_override: None,
9711            })
9712            .await
9713            .unwrap();
9714
9715        let mut saw_speed_policy_event = false;
9716        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9717            loop {
9718                let envelope = rx.recv().await.unwrap();
9719                if envelope.turn_id.as_deref() != Some(&turn_id) {
9720                    continue;
9721                }
9722                match envelope.event {
9723                    RoderEvent::InferenceStarted(event) => {
9724                        if event.speed_policy.is_some() {
9725                            saw_speed_policy_event = true;
9726                        }
9727                    }
9728                    RoderEvent::TurnCompleted(_) => break,
9729                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9730                    _ => {}
9731                }
9732            }
9733        })
9734        .await
9735        .unwrap();
9736
9737        let requests = requests.lock().unwrap().clone();
9738        assert!(saw_speed_policy_event);
9739        assert!(requests.len() >= 4);
9740        assert!(requests.iter().all(|request| {
9741            request.model.provider == roder_api::catalog::PROVIDER_MOCK
9742                && request.model.model == "gpt-5.5"
9743        }));
9744        assert_eq!(
9745            requests[0].runtime.speed_policy.as_ref().map(|d| d.phase),
9746            Some(roder_api::inference::SpeedPolicyPhase::Orientation)
9747        );
9748        assert_eq!(requests[0].reasoning.level.as_deref(), Some(REASONING_HIGH));
9749        assert_eq!(
9750            requests[1].runtime.speed_policy.as_ref().map(|d| d.phase),
9751            Some(roder_api::inference::SpeedPolicyPhase::Execution)
9752        );
9753        assert_eq!(requests[1].reasoning.level.as_deref(), Some(REASONING_LOW));
9754        assert_eq!(
9755            requests[2].runtime.speed_policy.as_ref().map(|d| d.phase),
9756            Some(roder_api::inference::SpeedPolicyPhase::Verification)
9757        );
9758        assert_eq!(requests[2].reasoning.level.as_deref(), Some(REASONING_HIGH));
9759        assert_eq!(
9760            requests[2]
9761                .metadata
9762                .pointer("/speedPolicy/phase")
9763                .and_then(serde_json::Value::as_str),
9764            Some("verification")
9765        );
9766    }
9767
9768    #[tokio::test]
9769    async fn deadline_turn_timeout_emits_partial_result_and_clears_active_turn() {
9770        let mut builder = ExtensionRegistryBuilder::new();
9771        builder.inference_engine(Arc::new(DeadlineEngine));
9772        let runtime = Arc::new(
9773            Runtime::new(
9774                builder.build().unwrap(),
9775                RuntimeConfig {
9776                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9777                    default_model: "mock".to_string(),
9778                    runtime_profile: RuntimeProfile::Eval,
9779                    turn_deadline_seconds: Some(1),
9780                    ..RuntimeConfig::default()
9781                },
9782            )
9783            .unwrap(),
9784        );
9785        let mut rx = runtime.subscribe_events();
9786        let turn_id = runtime
9787            .start_turn(StartTurnRequest {
9788                thread_id: "thread-deadline".to_string(),
9789                message: "slow work".to_string(),
9790                images: Vec::new(),
9791                provider_override: None,
9792                model_override: None,
9793                reasoning_override: None,
9794                workspace: test_workspace(),
9795                instructions: InstructionBundle::default(),
9796                developer_context: None,
9797                task_ledger_required: false,
9798                service_tier_override: None,
9799            })
9800            .await
9801            .unwrap();
9802
9803        let mut saw_partial = false;
9804        let mut saw_deadline = false;
9805        let mut failed_kind = None;
9806        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9807            loop {
9808                let envelope = rx.recv().await.unwrap();
9809                if envelope.turn_id.as_deref() != Some(&turn_id) {
9810                    continue;
9811                }
9812                match envelope.event {
9813                    RoderEvent::TurnPartialResult(event) => {
9814                        saw_partial = event.summary.contains("partial turn state");
9815                    }
9816                    RoderEvent::TurnDeadlineExceeded(event) => {
9817                        saw_deadline = event.partial_result.contains("transcript items");
9818                    }
9819                    RoderEvent::TurnFailed(event) => {
9820                        failed_kind = event.error_kind;
9821                        break;
9822                    }
9823                    _ => {}
9824                }
9825            }
9826        })
9827        .await
9828        .unwrap();
9829
9830        for _ in 0..20 {
9831            if !runtime.active_turns.read().await.contains_key(&turn_id) {
9832                break;
9833            }
9834            tokio::time::sleep(std::time::Duration::from_millis(10)).await;
9835        }
9836        assert!(saw_partial);
9837        assert!(saw_deadline);
9838        assert_eq!(failed_kind.as_deref(), Some("deadline_timeout"));
9839        assert!(!runtime.active_turns.read().await.contains_key(&turn_id));
9840    }
9841
9842    #[tokio::test]
9843    async fn deadline_skips_subagent_task_when_remaining_budget_is_too_low() {
9844        let calls = Arc::new(StdMutex::new(0));
9845        let mut builder = ExtensionRegistryBuilder::new();
9846        builder.inference_engine(Arc::new(CapturingEngine {
9847            request: Arc::new(StdMutex::new(None)),
9848        }));
9849        let task_tool = Arc::new(CountingTaskTool {
9850            calls: calls.clone(),
9851        });
9852        builder.tool_contributor(Arc::new(TestToolContributor { tool: task_tool }));
9853        let runtime = Arc::new(
9854            Runtime::new(
9855                builder.build().unwrap(),
9856                RuntimeConfig {
9857                    policy_mode: PolicyMode::Bypass,
9858                    agent_swarm_mode: false,
9859                    ultra_mode: false,
9860                    ..RuntimeConfig::default()
9861                },
9862            )
9863            .unwrap(),
9864        );
9865
9866        let result = runtime
9867            .route_tool_call(
9868                &"thread-deadline-task".to_string(),
9869                &"turn-deadline-task".to_string(),
9870                ToolCallCompleted {
9871                    id: "task-1".to_string(),
9872                    name: "task".to_string(),
9873                    arguments: serde_json::json!({
9874                        "description": "inspect",
9875                        "prompt": "read"
9876                    })
9877                    .to_string(),
9878                },
9879                None,
9880                Some(OffsetDateTime::now_utc() + Duration::seconds(1)),
9881                None,
9882            )
9883            .await
9884            .unwrap();
9885
9886        assert!(result.is_error);
9887        assert!(result.result.contains("deadline policy skipped"));
9888        assert_eq!(*calls.lock().unwrap(), 0);
9889    }
9890
9891    struct TestToolContributor {
9892        tool: Arc<dyn ToolExecutor>,
9893    }
9894
9895    impl ToolContributor for TestToolContributor {
9896        fn id(&self) -> String {
9897            "test-tool".to_string()
9898        }
9899
9900        fn contribute(&self, registry: &mut ToolRegistry) -> anyhow::Result<()> {
9901            registry.register(self.tool.clone())
9902        }
9903    }
9904
9905    #[tokio::test]
9906    async fn agent_swarm_mode_override_is_per_thread() {
9907        let runtime = Runtime::fake().unwrap();
9908        let trigger = roder_api::subagents::AgentSwarmModeTrigger::Manual;
9909
9910        // Off everywhere by default.
9911        assert!(
9912            !runtime
9913                .effective_agent_swarm_mode_for_thread("thread-a")
9914                .await
9915        );
9916        assert!(
9917            !runtime
9918                .effective_agent_swarm_mode_for_thread("thread-b")
9919                .await
9920        );
9921
9922        // Enabling on thread-a does not leak into thread-b.
9923        assert!(
9924            runtime
9925                .set_agent_swarm_mode_for_thread("thread-a", true, trigger)
9926                .await
9927        );
9928        assert!(
9929            runtime
9930                .effective_agent_swarm_mode_for_thread("thread-a")
9931                .await
9932        );
9933        assert!(
9934            !runtime
9935                .effective_agent_swarm_mode_for_thread("thread-b")
9936                .await
9937        );
9938
9939        // A per-thread `off` override wins over a runtime-global `on` default.
9940        runtime.set_agent_swarm_mode(true, trigger).await.unwrap();
9941        runtime
9942            .set_agent_swarm_mode_for_thread("thread-b", false, trigger)
9943            .await;
9944        assert!(
9945            !runtime
9946                .effective_agent_swarm_mode_for_thread("thread-b")
9947                .await,
9948            "explicit per-thread off overrides the global on default"
9949        );
9950        // A thread with no override still follows the global default.
9951        assert!(
9952            runtime
9953                .effective_agent_swarm_mode_for_thread("thread-c")
9954                .await,
9955            "threads without an override follow the runtime-global default"
9956        );
9957    }
9958
9959    #[tokio::test]
9960    async fn set_agent_swarm_mode_for_thread_emits_event_with_real_thread_id() {
9961        let runtime = Runtime::fake().unwrap();
9962        let mut events = runtime.subscribe_events();
9963        runtime
9964            .set_agent_swarm_mode_for_thread(
9965                "thread-xyz",
9966                true,
9967                roder_api::subagents::AgentSwarmModeTrigger::Task,
9968            )
9969            .await;
9970        let mut saw = false;
9971        for _ in 0..8 {
9972            let envelope = tokio::time::timeout(std::time::Duration::from_secs(2), events.recv())
9973                .await
9974                .unwrap()
9975                .unwrap();
9976            if let RoderEvent::AgentSwarmModeChanged(event) = envelope.event {
9977                assert_eq!(event.thread_id, "thread-xyz");
9978                assert!(event.enabled);
9979                assert_eq!(
9980                    event.trigger,
9981                    roder_api::subagents::AgentSwarmModeTrigger::Task
9982                );
9983                saw = true;
9984                break;
9985            }
9986        }
9987        assert!(
9988            saw,
9989            "expected an AgentSwarmModeChanged event for the thread"
9990        );
9991    }
9992
9993    #[tokio::test]
9994    async fn ultra_mode_override_is_per_thread() {
9995        let runtime = Runtime::fake().unwrap();
9996        let trigger = roder_api::subagents::UltraModeTrigger::Manual;
9997
9998        assert!(!runtime.effective_ultra_mode_for_thread("thread-a").await);
9999        assert!(!runtime.effective_ultra_mode_for_thread("thread-b").await);
10000
10001        assert!(
10002            runtime
10003                .set_ultra_mode_for_thread("thread-a", true, trigger)
10004                .await
10005        );
10006        assert!(runtime.effective_ultra_mode_for_thread("thread-a").await);
10007        assert!(!runtime.effective_ultra_mode_for_thread("thread-b").await);
10008
10009        runtime.set_ultra_mode(true, trigger).await.unwrap();
10010        runtime
10011            .set_ultra_mode_for_thread("thread-b", false, trigger)
10012            .await;
10013        assert!(
10014            !runtime.effective_ultra_mode_for_thread("thread-b").await,
10015            "explicit per-thread off overrides the global on default"
10016        );
10017        assert!(
10018            runtime.effective_ultra_mode_for_thread("thread-c").await,
10019            "threads without an override follow the runtime-global default"
10020        );
10021    }
10022
10023    #[tokio::test]
10024    async fn set_ultra_mode_for_thread_emits_event_with_real_thread_id() {
10025        let runtime = Runtime::fake().unwrap();
10026        let mut events = runtime.subscribe_events();
10027        runtime
10028            .set_ultra_mode_for_thread(
10029                "thread-ultra",
10030                true,
10031                roder_api::subagents::UltraModeTrigger::Task,
10032            )
10033            .await;
10034        let mut saw = false;
10035        for _ in 0..8 {
10036            let envelope = tokio::time::timeout(std::time::Duration::from_secs(2), events.recv())
10037                .await
10038                .unwrap()
10039                .unwrap();
10040            if let RoderEvent::UltraModeChanged(event) = envelope.event {
10041                assert_eq!(event.thread_id, "thread-ultra");
10042                assert!(event.enabled);
10043                assert_eq!(event.trigger, roder_api::subagents::UltraModeTrigger::Task);
10044                saw = true;
10045                break;
10046            }
10047        }
10048        assert!(saw, "expected an UltraModeChanged event for the thread");
10049    }
10050}
10051
10052#[cfg(test)]
10053#[path = "runtime/mailbox_test_helpers.rs"]
10054mod mailbox_test_helpers;