Skip to main content

roder_core/
runtime.rs

1#[path = "runtime/sampling.rs"]
2pub(crate) mod sampling;
3#[path = "tool_advertisement.rs"]
4mod tool_advertisement;
5use sampling::{
6    SamplingPreempted, SamplingRetry, completed_call_replayed, preemptible, wait_for_steer,
7};
8#[path = "runtime/sampling_output.rs"]
9mod sampling_output;
10use sampling_output::{OutputContext, SamplingOutput};
11mod compaction_template;
12#[path = "runtime/eager_tools.rs"]
13mod eager_tools;
14mod thread_admission;
15use eager_tools::EagerTools;
16#[path = "runtime/patch_progress.rs"]
17mod patch_progress;
18use patch_progress::PatchProgressTracker;
19
20use std::collections::{HashMap, HashSet};
21use std::path::PathBuf;
22use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
23use std::sync::{Arc, Weak};
24
25use anyhow::Context;
26use futures::StreamExt;
27use futures::future::{AbortHandle, Abortable, BoxFuture, try_join_all};
28use roder_api::catalog::{
29    EDIT_TOOL_EDIT, EDIT_TOOL_PATCH, PROVIDER_GEMINI, REASONING_NONE, REASONING_ULTRA,
30    built_in_model_profile, built_in_model_profile_for_provider, lookup_model,
31    model_supports_reasoning_effort,
32};
33use roder_api::context::PolicyGate;
34use roder_api::events::*;
35use roder_api::extension::ExtensionRegistry;
36use roder_api::inference::{
37    AgentInferenceRequest, HostedWebSearchConfig, HostedWebSearchMode, InferenceEngine,
38    InferenceEvent, InferenceTurnContext, InstructionBundle, ModelHarnessProfile,
39    ModelSchemaPolicy, ModelSelection, OutputConfig, ProviderTurnCleanup, ReasoningConfig,
40    RuntimeHints, RuntimeProfile, TokenUsage, ToolCallCompleted, ToolSearchConfig,
41    ToolSearchConfigOverlay, finish_reason_from_stop_reason,
42};
43use roder_api::inference_routing::{InferenceRoutingOutcome, ModelSelectionMode};
44use roder_api::lifecycle::{
45    LifecycleMetricsSnapshot, TurnCleanupOwnership, TurnCleanupState, TurnLifecycleReason,
46    TurnLifecycleRecord, TurnLifecycleSnapshot, TurnLifecycleState, turn_lifecycle_snapshot,
47};
48use roder_api::policy_mode::{PolicyDecision, PolicyMode};
49use roder_api::reliability::{
50    ReliabilityContext, ReliabilityDetails, ReliabilityErrorClass, ReliabilityLimitDecision,
51    ReliabilityLimitRecorded, ReliabilityRequestPolicy, ReliabilityRetryDecision,
52    ReliabilityRetryRecorded, provider_retry_delay_ms,
53};
54use roder_api::remote_runner::{
55    RemoteRunnerProvider, RemoteRunnerSession, RemoteWorkspace, RunnerDestination,
56    RunnerSessionState, ThreadRunnerBinding,
57};
58use roder_api::subagents::SubagentDefinition;
59use roder_api::teams::{
60    TeamId, TeamMailboxMessage, TeamMailboxMessageKind, TeamMemberDescriptor, TeamMemberRole,
61    TeamMemberStatus,
62};
63use roder_api::thread::{
64    ThreadItemEvent, ThreadItemEventKind, ThreadMetadata, ThreadSnapshot, ThreadStore,
65    ThreadUsageMetadata, is_synthetic_event_thread_id, validate_thread_workspace,
66};
67use roder_api::tools::{ToolCall, ToolChoice, ToolExecutionContext, ToolRegistry, ToolResult};
68use roder_api::transcript::{
69    AssistantMessage, ErrorRecord, InputImage, ReasoningSummary, ToolCallRecord, ToolResultRecord,
70    TranscriptItem, UserMessage,
71};
72use roder_sandbox::ScopedFilesystem;
73use roder_sandbox::process::LocalProcessRunner;
74use roder_skills::{SkillRegistry, SkillRegistryOptions};
75use time::{Duration, OffsetDateTime};
76use tokio::sync::{Mutex, Notify, RwLock, oneshot};
77
78mod codex_v2;
79
80use crate::artifacts::{
81    ContextArtifactStore as FilesystemContextArtifactStore, default_context_artifact_dir,
82};
83use crate::bus::EventBus;
84use crate::dynamic_workflows::{
85    DynamicWorkflowEffortProfile, RuntimeDynamicWorkflowConfig, WorkflowTriggerDecision,
86    classify_workflow_trigger, ultracode_reasoning_level_for_model,
87};
88use crate::fake_provider::FakeInferenceEngine;
89use crate::goals::RuntimeGoalController;
90use crate::inference_routing::{
91    InferenceRoutingRequest, RuntimeInferenceRouterConfig, collect_inference_routing_candidates,
92    route_inference_selection, transcript_failure_count_since,
93};
94use crate::instructions::{
95    apply_agent_swarm_mode, apply_codex_multi_agent_mode, apply_model_instruction_overlay,
96    apply_parallel_web_tools, apply_plan_mode, apply_runtime_profile, apply_task_ledger_required,
97    apply_thread_developer_instructions, apply_turn_developer_context,
98};
99use crate::policy_gate::DefaultPolicyGate;
100use crate::reliability::{
101    ReliabilityLimitHit, RuntimeReliabilityConfig, TurnReliabilityState,
102    provider_stream_retry_cause,
103};
104pub use crate::speed_policy::RuntimeSpeedPolicyConfig;
105use crate::speed_policy::{SpeedPolicyState, reasoning_from_decision};
106use crate::subagent_traces::{RuntimeAgentSwarmProgressSink, RuntimeSubagentTraceSink};
107use crate::teams::{TeamManager, TeamMemberStartRequest, TeamStartRequest, TeamState};
108use crate::thread_item_cache::{ThreadItemCache, ThreadItemCacheEntry};
109use crate::verification_gate::VerificationGateState;
110
111const MAX_TOOL_ROUNDS_PER_TURN: usize = 1024;
112/// Injected when `continue_on_failure_limit` turns a consecutive-tool-failure
113/// limit into a continuation, or as the empty-tool-call persistence nudge, so
114/// the model keeps working instead of ending the turn early.
115const RELIABILITY_CONTINUATION_PROMPT: &str = "Verify the task is fully complete. If any part remains unfinished or unverified, keep working using the available tools — try an alternative approach if one is failing. Only stop once the solution is complete and verified; if it is already complete, restate your final answer.";
116/// Capacity of the runtime event broadcast ring buffer. The TUI drains this on
117/// its render loop, which during an active turn only wakes at the ~6 FPS status
118/// animation cadence (backend events do not wake the input poll), so up to
119/// ~166ms of events buffer between drains. A reasoning-high provider that runs
120/// its whole tool loop inside one turn (e.g. claude-code) streams a firehose of
121/// `InferenceEventReceived` deltas plus per-step tool/thinking events; the old
122/// 1024-slot buffer overflowed in those windows and silently dropped tool and
123/// thinking rows from the live view. Sized for generous headroom across bursts
124/// and brief render stalls.
125const EVENT_BUS_CAPACITY: usize = 16_384;
126
127/// Turn id reported to session-scoped local hooks. `SessionStart` and
128/// `SessionEnd` bracket the session rather than any one turn, but the hook
129/// payload carries a turn id, so they share this synthetic value.
130const SESSION_HOOK_TURN_ID: &str = "session";
131pub(crate) const FINAL_ANSWER_PHASE: &str = "final_answer";
132pub(crate) const TASK_LEDGER_TOOL_NAME: &str = "task_ledger.update";
133const TASK_LEDGER_COMPLETION_REMINDER_LIMIT: u8 = 2;
134const TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS: u64 = 180;
135const TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT: u8 = 1;
136pub(crate) const MIN_CHILD_DEADLINE_SECONDS: u64 = 2;
137const MODEL_PROFILE_TRACE_KIND: &str = "model_profile_segment";
138const MODEL_SWITCH_SUMMARY_PREFIX: &str = "Model switch summary:";
139const PROVIDER_CLEANUP_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5);
140
141#[derive(Clone, Copy, Debug, Eq, PartialEq)]
142enum InferenceTimeoutAction {
143    ScoreableCheckpoint,
144    Finalization,
145}
146
147#[derive(Debug, Clone)]
148pub struct RuntimeConfig {
149    pub default_provider: String,
150    pub default_model: String,
151    pub reasoning: Option<String>,
152    pub auto_compact_token_limit: Option<u32>,
153    pub file_backed_dynamic_context: bool,
154    pub hosted_web_search: HostedWebSearchConfig,
155    pub tool_search: ToolSearchConfig,
156    pub provider_tool_search: HashMap<String, ToolSearchConfigOverlay>,
157    pub model_tool_search: HashMap<String, ToolSearchConfigOverlay>,
158    pub model_edit_tools: HashMap<String, String>,
159    pub model_parallel_tool_calls: HashMap<String, bool>,
160    pub model_profiles: HashMap<String, ModelHarnessProfile>,
161    pub tool_allowlist: Vec<String>,
162    /// Seconds a host-executed external tool call may stay unresolved before it fails with a timeout error.
163    pub external_tool_timeout_seconds: u64,
164    pub command_shell: String,
165    pub workspace: Option<String>,
166    pub policy_mode: PolicyMode,
167    /// Whether agent-swarm mode is active (roadmap 104). When on, the runtime
168    /// injects the swarm reminder into each turn's developer instructions so any
169    /// client benefits, not just the TUI.
170    pub agent_swarm_mode: bool,
171    /// Whether ultra mode is active. When on, the runtime injects proactive
172    /// multi-agent delegation policy for any model (not only Codex Sol/Terra
173    /// Ultra effort). Selecting Ultra reasoning on Sol/Terra still enables
174    /// proactive multi-agent for that model without requiring this flag.
175    pub ultra_mode: bool,
176    pub runtime_profile: RuntimeProfile,
177    pub inference_router: RuntimeInferenceRouterConfig,
178    pub speed_policy: RuntimeSpeedPolicyConfig,
179    pub dynamic_workflows: RuntimeDynamicWorkflowConfig,
180    pub reliability: RuntimeReliabilityConfig,
181    /// Positive wall-clock limit for a turn in any runtime profile. `None` or zero is unbounded.
182    pub turn_deadline_seconds: Option<u64>,
183    pub remote_runner_destination: Option<RunnerDestination>,
184    pub team_data_dir: Option<PathBuf>,
185    pub roadmap_data_dir: Option<PathBuf>,
186    pub media_generation: crate::media_generation::RuntimeMediaGenerationConfig,
187    /// `[review]` settings: the review sub-turn and its publishers.
188    pub review: crate::review::RuntimeReviewConfig,
189}
190
191impl Default for RuntimeConfig {
192    fn default() -> Self {
193        Self {
194            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
195            default_model: "mock".to_string(),
196            reasoning: None,
197            auto_compact_token_limit: None,
198            file_backed_dynamic_context: true,
199            hosted_web_search: HostedWebSearchConfig::cached(),
200            tool_search: ToolSearchConfig::default(),
201            provider_tool_search: HashMap::new(),
202            model_tool_search: HashMap::new(),
203            model_edit_tools: HashMap::new(),
204            model_parallel_tool_calls: HashMap::new(),
205            model_profiles: HashMap::new(),
206            tool_allowlist: Vec::new(),
207            external_tool_timeout_seconds: DEFAULT_EXTERNAL_TOOL_TIMEOUT_SECONDS,
208            command_shell: roder_api::command_shell::default_command_shell(),
209            workspace: None,
210            policy_mode: PolicyMode::Default,
211            agent_swarm_mode: false,
212            ultra_mode: false,
213            runtime_profile: RuntimeProfile::Interactive,
214            inference_router: RuntimeInferenceRouterConfig::default(),
215            speed_policy: RuntimeSpeedPolicyConfig::default(),
216            dynamic_workflows: RuntimeDynamicWorkflowConfig::default(),
217            reliability: RuntimeReliabilityConfig::default(),
218            turn_deadline_seconds: None,
219            remote_runner_destination: None,
220            team_data_dir: None,
221            roadmap_data_dir: None,
222            media_generation: crate::media_generation::RuntimeMediaGenerationConfig::default(),
223            review: crate::review::RuntimeReviewConfig::default(),
224        }
225    }
226}
227
228#[derive(Debug, Clone)]
229pub struct StartTurnRequest {
230    pub thread_id: ThreadId,
231    pub message: String,
232    pub images: Vec<InputImage>,
233    pub provider_override: Option<String>,
234    pub model_override: Option<String>,
235    pub reasoning_override: Option<String>,
236    pub workspace: String,
237    pub instructions: InstructionBundle,
238    /**
239     * Per-turn developer-authority context for this turn's InstructionBundle.
240     * Applies to every inference round of the turn, is never written to
241     * thread state, and does not carry over to later turns.
242     */
243    pub developer_context: Option<String>,
244    pub task_ledger_required: bool,
245    /**
246     * Per-turn provider service tier (OpenAI `service_tier`, e.g.
247     * `"priority"` for Fast mode). Carried to every inference round of the
248     * turn as `RuntimeHints::service_tier`; `None` keeps the provider
249     * default. Providers without a tier concept ignore it.
250     */
251    pub service_tier_override: Option<String>,
252}
253
254#[derive(Debug, Clone)]
255pub struct CreateThreadRequest {
256    pub title: Option<String>,
257    pub workspace: String,
258    pub workspace_id: Option<String>,
259    pub root_id: Option<String>,
260    pub provider: Option<String>,
261    pub model: Option<String>,
262    pub selection_mode: Option<ModelSelectionMode>,
263    /// Per-thread tool filter applied on top of the runtime allowlist. Empty = no filtering.
264    pub tool_allowlist: Vec<String>,
265    /// Host-supplied instructions added to the developer slot of every turn's inference request.
266    pub developer_instructions: Option<String>,
267    /// Host-executed tool specs advertised to the model on every turn of this thread.
268    pub external_tools: Vec<roder_api::tools::ToolSpec>,
269    /// Explicit remote-runner binding for the thread's native coding tools.
270    pub runner: Option<ThreadRunnerSelection>,
271}
272
273/**
274 * Thread-level remote-runner selection. The destination config is persisted
275 * with the thread, so secrets must reach the provider through its
276 * environment, not this config.
277 */
278#[derive(Debug, Clone)]
279pub struct ThreadRunnerSelection {
280    pub provider_id: String,
281    pub config: serde_json::Value,
282    /// Absolute path on the runner used as the thread's coding-tool workspace root.
283    pub workspace: String,
284    /**
285     * Extra absolute runner paths file reads may resolve under, beyond
286     * `workspace`. Writes and the working directory stay confined to
287     * `workspace`.
288     */
289    pub read_roots: Vec<String>,
290}
291
292#[derive(Debug, Clone, PartialEq, Eq)]
293pub struct PendingPlanExit {
294    pub thread_id: ThreadId,
295    pub turn_id: TurnId,
296    pub request_id: String,
297    pub target_mode: PolicyMode,
298    pub plan_summary: Option<String>,
299    pub next_steps: Vec<String>,
300    pub requested_at: OffsetDateTime,
301    pub expires_at: Option<OffsetDateTime>,
302}
303
304pub(crate) struct PendingToolApproval {
305    pub(crate) thread_id: ThreadId,
306    pub(crate) turn_id: TurnId,
307    pub(crate) tool_id: String,
308    pub(crate) tool_name: String,
309    pub(crate) call: roder_api::tools::ToolCall,
310    pub(crate) tx: oneshot::Sender<bool>,
311}
312
313pub(crate) struct PendingUserInput {
314    pub(crate) thread_id: ThreadId,
315    pub(crate) turn_id: TurnId,
316    pub(crate) tx: oneshot::Sender<serde_json::Value>,
317}
318
319/// Host answer to an external tool call delivered via `tools/resolve`.
320#[derive(Debug, Clone, PartialEq, Eq)]
321pub struct ExternalToolResolution {
322    pub output: String,
323    pub is_error: bool,
324}
325
326pub(crate) struct PendingExternalToolCall {
327    pub(crate) thread_id: ThreadId,
328    pub(crate) turn_id: TurnId,
329    pub(crate) tool_id: String,
330    pub(crate) tool_name: String,
331    pub(crate) tx: oneshot::Sender<ExternalToolResolution>,
332}
333
334#[derive(Clone)]
335struct ActiveTurnHandle {
336    thread_id: ThreadId,
337    abort: AbortHandle,
338    steers: Arc<Mutex<Vec<QueuedTurnSteer>>>,
339    steer_changed: tokio::sync::watch::Sender<u64>,
340    drain: Arc<TurnDrainHandle>,
341}
342
343/// Tracks a task after it has been interrupted and removed from the interactive
344/// active-turn map. This lets users start a follow-up turn immediately while a
345/// bounded runtime shutdown can still await the original task's cleanup.
346struct TurnDrainHandle {
347    thread_id: ThreadId,
348    interrupt_requested: AtomicBool,
349    interrupt_reason: Mutex<Option<TurnLifecycleReason>>,
350    completed: AtomicBool,
351    completed_notify: Notify,
352}
353
354/// Result of a bounded runtime drain. `Clean` proves the runtime's own turn
355/// tasks have reached their cleanup paths; provider-specific child-process
356/// reaping remains an explicit provider capability.
357#[derive(Debug, Clone, PartialEq, Eq)]
358pub enum RuntimeDrainOutcome {
359    Clean {
360        interrupted_turn_ids: Vec<TurnId>,
361    },
362    DeadlineExceeded {
363        interrupted_turn_ids: Vec<TurnId>,
364        remaining_turn_ids: Vec<TurnId>,
365    },
366    /// At least one lifecycle transition initiated by this drain could not be
367    /// durably appended. The runtime still made its best cleanup attempt, but
368    /// callers must not claim a persisted terminal state as proof of recovery.
369    PersistenceFailed {
370        interrupted_turn_ids: Vec<TurnId>,
371        remaining_turn_ids: Vec<TurnId>,
372    },
373}
374
375#[derive(Clone)]
376struct QueuedTurnSteer {
377    message: UserMessage,
378    mailbox_ack: Option<MailboxDeliveryAck>,
379}
380
381#[derive(Clone)]
382struct MailboxDeliveryAck {
383    team_id: TeamId,
384    message_ids: Vec<String>,
385}
386
387#[derive(Clone)]
388struct InheritedTurnContext {
389    workspace: String,
390    instructions: InstructionBundle,
391    developer_context: Option<String>,
392}
393
394#[derive(Debug, Clone, Default, PartialEq, Eq)]
395pub struct ThreadActivity {
396    pub active_turn_id: Option<TurnId>,
397    pub active_flags: Vec<String>,
398}
399
400/// Per-thread settings persisted at thread creation and applied to every turn.
401#[derive(Debug, Clone, Default)]
402pub(crate) struct ThreadTurnOverrides {
403    pub(crate) tool_allowlist: Vec<String>,
404    pub(crate) developer_instructions: Option<String>,
405    pub(crate) external_tools: Vec<roder_api::tools::ToolSpec>,
406}
407
408#[derive(Debug, Clone, Copy, PartialEq, Eq)]
409pub(crate) enum TurnRunOutcome {
410    Completed,
411    Stopped,
412}
413
414impl PendingPlanExit {
415    pub fn new(
416        thread_id: ThreadId,
417        turn_id: TurnId,
418        request_id: String,
419        target_mode: PolicyMode,
420        plan_summary: Option<String>,
421        next_steps: Vec<String>,
422    ) -> Self {
423        let requested_at = OffsetDateTime::now_utc();
424        Self {
425            thread_id,
426            turn_id,
427            request_id,
428            target_mode,
429            plan_summary,
430            next_steps,
431            requested_at,
432            expires_at: Some(requested_at + default_plan_exit_timeout()),
433        }
434    }
435
436    pub fn is_expired(&self, now: OffsetDateTime) -> bool {
437        self.expires_at.is_some_and(|expires_at| now >= expires_at)
438    }
439}
440
441pub fn default_plan_exit_timeout() -> Duration {
442    Duration::minutes(10)
443}
444
445pub const DEFAULT_EXTERNAL_TOOL_TIMEOUT_SECONDS: u64 = 300;
446
447fn format_mailbox_messages(team: &TeamState, messages: &[TeamMailboxMessage]) -> String {
448    messages
449        .iter()
450        .map(|message| {
451            let recipient = team
452                .members
453                .iter()
454                .find(|member| member.id == message.to_member_id)
455                .map(canonical_team_member_path)
456                .unwrap_or_else(|| format!("/root/{}", message.to_member_id));
457            let sender = message
458                .from_member_id
459                .as_deref()
460                .and_then(|id| team.members.iter().find(|member| member.id == id))
461                .map(canonical_team_member_path)
462                .unwrap_or_else(|| "/root".to_string());
463            let message_type = match message.kind {
464                TeamMailboxMessageKind::Message => "MESSAGE",
465                TeamMailboxMessageKind::NewTask => "NEW_TASK",
466                TeamMailboxMessageKind::FinalAnswer => "FINAL_ANSWER",
467            };
468            format!(
469                "Message Type: {message_type}\nTask name: {recipient}\nSender: {sender}\nPayload:\n{}",
470                message.text
471            )
472        })
473        .collect::<Vec<_>>()
474        .join("\n\n")
475}
476
477fn canonical_team_member_path(member: &TeamMemberDescriptor) -> String {
478    if let Some(agent_path) = member
479        .agent_path
480        .as_deref()
481        .filter(|path| *path == "/root" || path.starts_with("/root/"))
482    {
483        return agent_path.to_string();
484    }
485    if member.role == TeamMemberRole::Lead {
486        return "/root".to_string();
487    }
488    member
489        .task_name
490        .as_deref()
491        .filter(|name| !name.trim().is_empty())
492        .map(|name| format!("/root/{}", name.trim().trim_matches('/')))
493        .unwrap_or_else(|| format!("/root/{}", member.id))
494}
495
496fn is_codex_v2_team(team: &TeamState) -> bool {
497    team.members.iter().any(|member| {
498        member
499            .model
500            .as_deref()
501            .is_some_and(|model| model_supports_reasoning_effort(model, REASONING_ULTRA))
502    })
503}
504
505fn runtime_local_team_view(
506    mut team: TeamState,
507    active_thread_ids: &std::collections::HashSet<ThreadId>,
508) -> TeamState {
509    for member in &mut team.members {
510        if member.status == TeamMemberStatus::Running
511            && !active_thread_ids.contains(&member.thread_id)
512        {
513            member.status = TeamMemberStatus::Interrupted;
514            member.current_turn_id = None;
515        }
516    }
517    team
518}
519
520type RunnerToolExecutionKey = (String, String);
521type RunnerToolExecutionMutex = Mutex<()>;
522type RunnerToolExecutionLockRegistry =
523    Mutex<HashMap<RunnerToolExecutionKey, Weak<RunnerToolExecutionMutex>>>;
524
525pub struct Runtime {
526    pub bus: EventBus,
527    pub registry: ExtensionRegistry,
528    config: RwLock<RuntimeConfig>,
529    pending_plan_exit: RwLock<Option<PendingPlanExit>>,
530    pub(crate) pending_tool_approvals: Mutex<HashMap<String, PendingToolApproval>>,
531    pub(crate) pending_user_inputs: Mutex<HashMap<String, PendingUserInput>>,
532    pub(crate) pending_external_tool_calls: Mutex<HashMap<String, PendingExternalToolCall>>,
533    /// Serializes turn admission with shutdown quiescing. A drain takes this
534    /// gate before flipping `accepting_turns` and snapshotting active work so a
535    /// turn that observed the old flag cannot be inserted after the snapshot.
536    turn_admission: Mutex<()>,
537    active_turns: RwLock<HashMap<TurnId, ActiveTurnHandle>>,
538    turn_drains: RwLock<HashMap<TurnId, Arc<TurnDrainHandle>>>,
539    /// Provider cleanup handles register synchronously through the inference
540    /// tool-executor context. They are intentionally separate from active turn
541    /// admission so an interrupted turn can remain drainable after a follow-up
542    /// turn starts on the same thread.
543    provider_turn_cleanups: std::sync::Mutex<HashMap<TurnId, Arc<dyn ProviderTurnCleanup>>>,
544    accepting_turns: AtomicBool,
545    /// Monotonic counter used by bounded drains to surface lifecycle-state
546    /// persistence failures without making an individual provider turn fail.
547    lifecycle_persistence_failures: AtomicUsize,
548    lifecycle_shutdown_drains: AtomicUsize,
549    lifecycle_clean_shutdowns: AtomicUsize,
550    lifecycle_deadline_exceeded: AtomicUsize,
551    lifecycle_persistence_failed_drains: AtomicUsize,
552    lifecycle_restart_reconciliations: AtomicUsize,
553    lifecycle_shutdown_drain_duration_ms_total: AtomicUsize,
554    provider_cleanup_confirmed: AtomicUsize,
555    provider_cleanup_timed_out: AtomicUsize,
556    provider_cleanup_unknown: AtomicUsize,
557    active_turns_changed: Notify,
558    active_turn_selections: RwLock<HashMap<TurnId, ModelSelectionMode>>,
559    compaction_templates: RwLock<HashMap<ThreadId, AgentInferenceRequest>>,
560    thread_admission_gates: Mutex<HashMap<ThreadId, Arc<Mutex<()>>>>,
561    /// Threads that have already dispatched a `SessionStart` local hook.
562    ///
563    /// `ThreadCreated` fires once, but `ThreadLoaded` fires on every read of the
564    /// thread from its store, so without this a session ran its setup hooks
565    /// several times — including after the turn had already stopped.
566    session_hook_started: RwLock<HashSet<ThreadId>>,
567    active_turn_contexts: RwLock<HashMap<TurnId, InheritedTurnContext>>,
568    /// Process-local execution boundary. Local/CLI runtimes default to true;
569    /// hosted runtime pools set this false so missing runner metadata fails
570    /// closed instead of exposing the host workspace.
571    allow_local_workspaces: AtomicBool,
572    // Spawn-time live authority retained for every reusable turn of a long-lived teammate.
573    // This stays process-local because developer_context is explicitly volatile and must never
574    // be persisted to thread state.
575    team_member_turn_contexts: Mutex<HashMap<ThreadId, InheritedTurnContext>>,
576    workspace: PathBuf,
577    pub(crate) teams: TeamManager,
578    agent_team_spawn_lock: Mutex<()>,
579    pub(crate) roadmaps: Mutex<roder_roadmap::RoadmapRuntime>,
580    /// Completed reviews, most recent last, so a later publish can resolve a
581    /// review id back to its findings. Bounded; see `review::run`.
582    pub(crate) reviews: Mutex<Vec<crate::review::ReviewRecord>>,
583    pub(crate) goals: Arc<RuntimeGoalController>,
584    context_artifacts: roder_api::artifacts::ContextArtifactStore,
585    pub(crate) thread_store: Option<Arc<dyn ThreadStore>>,
586    thread_item_cache: Mutex<ThreadItemCache>,
587    pub(crate) tool_registry: ToolRegistry,
588    media_generation: Arc<crate::media_generation::MediaGenerationService>,
589    pub(crate) skills: RwLock<SkillRegistry>,
590    /// Lazily-started bounded dispatch of emitted events to registry
591    /// `EventSink`s (process extensions etc.); see `event_sink_dispatch`.
592    event_sink_dispatcher: tokio::sync::OnceCell<crate::event_sink_dispatch::EventSinkDispatcher>,
593    pub(crate) compaction_hysteresis:
594        std::sync::Mutex<HashMap<ThreadId, crate::compaction_runtime::CompactionState>>,
595    /// Per-thread agent-swarm mode overrides (roadmap 104). A thread present in
596    /// this map uses its stored value; absent threads fall back to the
597    /// runtime-global `RuntimeConfig.agent_swarm_mode` default. This mirrors the
598    /// team per-member policy-mode override idiom so swarm mode is per-thread
599    /// like the other `thread/*` operations, without a separate runtime.
600    agent_swarm_modes: RwLock<HashMap<ThreadId, bool>>,
601    /// Per-thread ultra mode overrides. A thread present in this map uses its
602    /// stored value; absent threads fall back to the runtime-global
603    /// `RuntimeConfig.ultra_mode` default.
604    ultra_modes: RwLock<HashMap<ThreadId, bool>>,
605    /**
606     * Live remote-runner sessions keyed by thread. Each per-thread mutex is
607     * held across provider initialization, making the first workspace-tool
608     * access a singleflight while allowing unrelated threads to initialize in
609     * parallel. Failed initialization is shared with current waiters, then
610     * evicted so a later tool call can retry.
611     */
612    runner_sessions: Arc<Mutex<HashMap<ThreadId, Arc<RunnerSessionSlot>>>>,
613    /**
614     * Outer tool-call locks keyed by the actual remote session, not the
615     * per-thread destination id. Hosted runtimes are tenant-scoped, so this
616     * serializes threads that share one sandbox without coupling different
617     * tenants or independent runner sessions.
618     */
619    runner_tool_execution_locks: RunnerToolExecutionLockRegistry,
620}
621
622#[derive(Clone)]
623struct CachedRunnerSession {
624    destination: RunnerDestination,
625    session: Arc<dyn RemoteRunnerSession>,
626}
627
628struct RunnerSessionSlot {
629    provider_id: String,
630    destination_id: String,
631    state: Mutex<RunnerSessionSlotState>,
632    initialized: Notify,
633}
634
635enum RunnerSessionSlotState {
636    Empty,
637    Initializing,
638    Ready(CachedRunnerSession),
639    Failed(Arc<str>),
640}
641
642impl RunnerSessionSlot {
643    fn empty(destination: &RunnerDestination) -> Self {
644        Self {
645            provider_id: destination.provider_id.clone(),
646            destination_id: destination.id.clone(),
647            state: Mutex::new(RunnerSessionSlotState::Empty),
648            initialized: Notify::new(),
649        }
650    }
651
652    fn ready(session: CachedRunnerSession) -> Self {
653        Self {
654            provider_id: session.destination.provider_id.clone(),
655            destination_id: session.destination.id.clone(),
656            state: Mutex::new(RunnerSessionSlotState::Ready(session)),
657            initialized: Notify::new(),
658        }
659    }
660
661    fn matches(&self, destination: &RunnerDestination) -> bool {
662        self.provider_id == destination.provider_id && self.destination_id == destination.id
663    }
664}
665
666impl Runtime {
667    pub fn new(registry: ExtensionRegistry, config: RuntimeConfig) -> anyhow::Result<Self> {
668        if registry.inference_engines.is_empty() {
669            anyhow::bail!("at least one inference engine must be registered");
670        }
671        validate_runtime_config_reasoning(&config)?;
672        validate_runtime_inference_router_config(&registry, &config)?;
673
674        let bus = EventBus::new(EVENT_BUS_CAPACITY);
675        let thread_store = registry
676            .thread_stores
677            .first()
678            .map(|factory| factory.create());
679        let mut tool_registry = ToolRegistry::default();
680        for contributor in &registry.tools {
681            contributor
682                .contribute(&mut tool_registry)
683                .with_context(|| format!("tool contributor {} failed", contributor.id()))?;
684        }
685        crate::agent_control_tools::contribute_agent_control_tools(&mut tool_registry)?;
686
687        let media_generation = Arc::new(crate::media_generation::MediaGenerationService::new(
688            registry.media_generator_providers.clone(),
689            config.media_generation.clone(),
690        ));
691        tool_registry.replace(Arc::new(
692            crate::media_generation::MediaGenerateImageTool::new(media_generation.clone()),
693        ));
694
695        let team_data_dir = config.team_data_dir.clone();
696        let workspace = config
697            .workspace
698            .clone()
699            .map(PathBuf::from)
700            .unwrap_or(std::env::current_dir()?);
701        let roadmap_data_dir = config
702            .roadmap_data_dir
703            .clone()
704            .unwrap_or_else(|| workspace.join(".roder"));
705        let context_artifacts = thread_store
706            .as_ref()
707            .and_then(|store| store.context_artifact_store())
708            .or_else(|| {
709                thread_store
710                    .as_ref()
711                    .and_then(|store| store.local_thread_root())
712                    .map(FilesystemContextArtifactStore::shared_thread_scoped)
713            })
714            .unwrap_or_else(|| {
715                FilesystemContextArtifactStore::shared_legacy(default_context_artifact_dir())
716            });
717        let goals = Arc::new(RuntimeGoalController::new(
718            bus.clone(),
719            thread_store.clone(),
720        ));
721        let runtime = Self {
722            bus,
723            registry,
724            config: RwLock::new(config),
725            pending_plan_exit: RwLock::new(None),
726            pending_tool_approvals: Mutex::new(HashMap::new()),
727            pending_user_inputs: Mutex::new(HashMap::new()),
728            pending_external_tool_calls: Mutex::new(HashMap::new()),
729            turn_admission: Mutex::new(()),
730            active_turns: RwLock::new(HashMap::new()),
731            turn_drains: RwLock::new(HashMap::new()),
732            provider_turn_cleanups: std::sync::Mutex::new(HashMap::new()),
733            accepting_turns: AtomicBool::new(true),
734            lifecycle_persistence_failures: AtomicUsize::new(0),
735            lifecycle_shutdown_drains: AtomicUsize::new(0),
736            lifecycle_clean_shutdowns: AtomicUsize::new(0),
737            lifecycle_deadline_exceeded: AtomicUsize::new(0),
738            lifecycle_persistence_failed_drains: AtomicUsize::new(0),
739            lifecycle_restart_reconciliations: AtomicUsize::new(0),
740            lifecycle_shutdown_drain_duration_ms_total: AtomicUsize::new(0),
741            provider_cleanup_confirmed: AtomicUsize::new(0),
742            provider_cleanup_timed_out: AtomicUsize::new(0),
743            provider_cleanup_unknown: AtomicUsize::new(0),
744            active_turns_changed: Notify::new(),
745            active_turn_selections: RwLock::new(HashMap::new()),
746            compaction_templates: RwLock::new(HashMap::new()),
747            thread_admission_gates: Mutex::new(HashMap::new()),
748            session_hook_started: RwLock::new(HashSet::new()),
749            active_turn_contexts: RwLock::new(HashMap::new()),
750            allow_local_workspaces: AtomicBool::new(true),
751            team_member_turn_contexts: Mutex::new(HashMap::new()),
752            workspace: workspace.clone(),
753            teams: TeamManager::new(
754                team_data_dir.unwrap_or_else(crate::teams::default_team_data_dir),
755            ),
756            agent_team_spawn_lock: Mutex::new(()),
757            roadmaps: Mutex::new(roder_roadmap::RoadmapRuntime::new(
758                workspace,
759                roadmap_data_dir,
760            )),
761            reviews: Mutex::new(Vec::new()),
762            goals,
763            context_artifacts,
764            thread_store,
765            thread_item_cache: Mutex::new(ThreadItemCache::default()),
766            tool_registry,
767            media_generation,
768            skills: RwLock::new(SkillRegistry::load(SkillRegistryOptions::new(
769                PathBuf::new(),
770            ))),
771            event_sink_dispatcher: tokio::sync::OnceCell::new(),
772            compaction_hysteresis: crate::compaction_runtime::compaction_hysteresis_state(),
773            agent_swarm_modes: RwLock::new(HashMap::new()),
774            ultra_modes: RwLock::new(HashMap::new()),
775            runner_sessions: Arc::new(Mutex::new(HashMap::new())),
776            runner_tool_execution_locks: Mutex::new(HashMap::new()),
777        };
778        runtime.bus.emit(RoderEvent::RuntimeStarted(RuntimeStarted {
779            timestamp: OffsetDateTime::now_utc(),
780        }));
781        for manifest in &runtime.registry.manifests {
782            runtime
783                .bus
784                .emit(RoderEvent::ExtensionRegistered(ExtensionRegistered {
785                    extension_id: manifest.id.clone(),
786                    timestamp: OffsetDateTime::now_utc(),
787                }));
788        }
789        Ok(runtime)
790    }
791
792    pub fn from_engine(engine: Arc<dyn InferenceEngine>) -> anyhow::Result<Self> {
793        let mut builder = roder_api::extension::ExtensionRegistryBuilder::new();
794        builder.inference_engine(engine);
795        Self::new(builder.build()?, RuntimeConfig::default())
796    }
797
798    pub fn fake() -> anyhow::Result<Self> {
799        Self::from_engine(Arc::new(FakeInferenceEngine))
800    }
801
802    pub fn subscribe_events(&self) -> tokio::sync::broadcast::Receiver<EventEnvelope> {
803        self.bus.subscribe()
804    }
805
806    /// Returns process-local, redacted lifecycle health counters. The snapshot
807    /// intentionally has fixed fields and contains no provider, command,
808    /// process, thread, or turn identifiers.
809    pub fn lifecycle_metrics(&self) -> LifecycleMetricsSnapshot {
810        LifecycleMetricsSnapshot {
811            shutdown_drain_count: self.lifecycle_shutdown_drains.load(Ordering::Acquire) as u64,
812            clean_shutdown_count: self.lifecycle_clean_shutdowns.load(Ordering::Acquire) as u64,
813            deadline_exceeded_count: self.lifecycle_deadline_exceeded.load(Ordering::Acquire)
814                as u64,
815            persistence_failed_count: self
816                .lifecycle_persistence_failed_drains
817                .load(Ordering::Acquire) as u64,
818            restart_reconciliation_count: self
819                .lifecycle_restart_reconciliations
820                .load(Ordering::Acquire) as u64,
821            lifecycle_persistence_failure_count: self
822                .lifecycle_persistence_failures
823                .load(Ordering::Acquire) as u64,
824            shutdown_drain_duration_ms_total: self
825                .lifecycle_shutdown_drain_duration_ms_total
826                .load(Ordering::Acquire) as u64,
827            provider_cleanup_confirmed_count: self
828                .provider_cleanup_confirmed
829                .load(Ordering::Acquire) as u64,
830            provider_cleanup_timed_out_count: self
831                .provider_cleanup_timed_out
832                .load(Ordering::Acquire) as u64,
833            provider_cleanup_unknown_count: self.provider_cleanup_unknown.load(Ordering::Acquire)
834                as u64,
835        }
836    }
837
838    async fn persist_turn_lifecycle_record(&self, record: &TurnLifecycleRecord) -> bool {
839        let Some(store) = &self.thread_store else {
840            return true;
841        };
842        let state = match record.extension_state() {
843            Ok(state) => state,
844            Err(_) => {
845                self.lifecycle_persistence_failures
846                    .fetch_add(1, Ordering::AcqRel);
847                return false;
848            }
849        };
850        // Lifecycle diagnostics must not turn a provider result into a failed
851        // turn merely because a third-party store lacks extension-state support.
852        // JSONL/Postgres stores persist this append-only record atomically at
853        // their own storage boundary.
854        if store
855            .append_extension_state(&record.thread_id, &state)
856            .await
857            .is_err()
858        {
859            self.lifecycle_persistence_failures
860                .fetch_add(1, Ordering::AcqRel);
861            return false;
862        }
863        true
864    }
865
866    async fn record_turn_lifecycle(
867        &self,
868        thread_id: ThreadId,
869        turn_id: TurnId,
870        state: TurnLifecycleState,
871        cleanup: TurnCleanupState,
872        reason: Option<TurnLifecycleReason>,
873    ) -> TurnLifecycleRecord {
874        self.record_turn_lifecycle_with_ownership(
875            thread_id,
876            turn_id,
877            state,
878            cleanup,
879            reason,
880            TurnCleanupOwnership::RuntimeTaskOnly,
881        )
882        .await
883    }
884
885    async fn record_turn_lifecycle_with_ownership(
886        &self,
887        thread_id: ThreadId,
888        turn_id: TurnId,
889        state: TurnLifecycleState,
890        cleanup: TurnCleanupState,
891        reason: Option<TurnLifecycleReason>,
892        ownership: TurnCleanupOwnership,
893    ) -> TurnLifecycleRecord {
894        let record = TurnLifecycleRecord::new(
895            thread_id,
896            turn_id,
897            state,
898            cleanup,
899            reason,
900            OffsetDateTime::now_utc(),
901        )
902        .with_ownership(ownership);
903        let _ = self.persist_turn_lifecycle_record(&record).await;
904        self.emit(RoderEvent::TurnLifecycleUpdated(record.clone()))
905            .await;
906        record
907    }
908
909    pub(crate) fn register_provider_turn_cleanup(
910        &self,
911        turn_id: &TurnId,
912        cleanup: Arc<dyn ProviderTurnCleanup>,
913    ) {
914        if let Ok(mut cleanups) = self.provider_turn_cleanups.lock() {
915            cleanups.insert(turn_id.clone(), cleanup);
916        }
917    }
918
919    fn provider_cleanup_ownership(&self, turn_id: &TurnId) -> TurnCleanupOwnership {
920        self.provider_turn_cleanups
921            .lock()
922            .ok()
923            .and_then(|cleanups| cleanups.get(turn_id).cloned())
924            .map(|cleanup| cleanup.ownership())
925            .unwrap_or(TurnCleanupOwnership::RuntimeTaskOnly)
926    }
927
928    async fn await_provider_turn_cleanup(
929        &self,
930        turn_id: &TurnId,
931    ) -> (TurnCleanupState, TurnCleanupOwnership) {
932        let cleanup = self
933            .provider_turn_cleanups
934            .lock()
935            .ok()
936            .and_then(|mut cleanups| cleanups.remove(turn_id));
937        let Some(cleanup) = cleanup else {
938            return (
939                TurnCleanupState::Unknown,
940                TurnCleanupOwnership::RuntimeTaskOnly,
941            );
942        };
943        let ownership = cleanup.ownership();
944        match tokio::time::timeout(PROVIDER_CLEANUP_TIMEOUT, cleanup.wait_for_cleanup()).await {
945            Ok(Ok(())) => {
946                self.provider_cleanup_confirmed
947                    .fetch_add(1, Ordering::AcqRel);
948                (
949                    TurnCleanupState::Completed,
950                    TurnCleanupOwnership::ProviderCleanupConfirmed,
951                )
952            }
953            Ok(Err(_)) => {
954                self.provider_cleanup_unknown.fetch_add(1, Ordering::AcqRel);
955                (TurnCleanupState::Unknown, ownership)
956            }
957            Err(_) => {
958                self.provider_cleanup_timed_out
959                    .fetch_add(1, Ordering::AcqRel);
960                (TurnCleanupState::TimedOut, ownership)
961            }
962        }
963    }
964
965    fn record_lifecycle_drain_outcome(
966        &self,
967        started_at: tokio::time::Instant,
968        outcome: &RuntimeDrainOutcome,
969    ) {
970        self.lifecycle_shutdown_drains
971            .fetch_add(1, Ordering::AcqRel);
972        self.lifecycle_shutdown_drain_duration_ms_total.fetch_add(
973            started_at.elapsed().as_millis().min(usize::MAX as u128) as usize,
974            Ordering::AcqRel,
975        );
976        match outcome {
977            RuntimeDrainOutcome::Clean { .. } => {
978                self.lifecycle_clean_shutdowns
979                    .fetch_add(1, Ordering::AcqRel);
980            }
981            RuntimeDrainOutcome::DeadlineExceeded { .. } => {
982                self.lifecycle_deadline_exceeded
983                    .fetch_add(1, Ordering::AcqRel);
984            }
985            RuntimeDrainOutcome::PersistenceFailed { .. } => {
986                self.lifecycle_persistence_failed_drains
987                    .fetch_add(1, Ordering::AcqRel);
988            }
989        }
990    }
991
992    fn lifecycle_record_for_event(event: &RoderEvent) -> Option<TurnLifecycleRecord> {
993        match event {
994            RoderEvent::TurnStarted(event) => Some(TurnLifecycleRecord::new(
995                event.thread_id.clone(),
996                event.turn_id.clone(),
997                TurnLifecycleState::Running,
998                TurnCleanupState::NotRequested,
999                None,
1000                event.timestamp,
1001            )),
1002            // `run_turn` persists completed turns after it has awaited a
1003            // registered provider cleanup handle. Do not let the generic event
1004            // projection overwrite that acknowledgement with an unproven state.
1005            RoderEvent::TurnCompleted(_) => None,
1006            // Some failure paths already have an event before the turn wrapper
1007            // reaches its cleanup acknowledgement. Preserve a durable fallback
1008            // reason here; the wrapper appends a later record with the more
1009            // precise cleanup outcome when a provider registered one.
1010            RoderEvent::TurnFailed(event) => Some(TurnLifecycleRecord::new(
1011                event.thread_id.clone(),
1012                event.turn_id.clone(),
1013                TurnLifecycleState::Failed,
1014                TurnCleanupState::Unknown,
1015                Some(if event.error_kind.as_deref() == Some("deadline_timeout") {
1016                    TurnLifecycleReason::DeadlineExceeded
1017                } else {
1018                    TurnLifecycleReason::ProviderFailure
1019                }),
1020                event.timestamp,
1021            )),
1022            RoderEvent::TurnDeadlineExceeded(_) | RoderEvent::TurnInterrupted(_) => None,
1023            RoderEvent::TurnLifecycleUpdated(_) => None,
1024            _ => None,
1025        }
1026    }
1027
1028    pub fn registry(&self) -> &ExtensionRegistry {
1029        &self.registry
1030    }
1031
1032    pub fn media_generation(&self) -> Arc<crate::media_generation::MediaGenerationService> {
1033        self.media_generation.clone()
1034    }
1035
1036    pub fn context_artifacts(&self) -> roder_api::artifacts::ContextArtifactStore {
1037        self.context_artifacts.clone()
1038    }
1039
1040    pub async fn execute_workflow_tool(
1041        &self,
1042        thread_id: ThreadId,
1043        tool_name: &str,
1044        arguments: serde_json::Value,
1045    ) -> anyhow::Result<ToolResult> {
1046        let Some(executor) = self.tool_registry.get(tool_name) else {
1047            anyhow::bail!("tool not found: {tool_name}");
1048        };
1049        let tool_call = ToolCall {
1050            id: format!("slash-{tool_name}"),
1051            name: tool_name.to_string(),
1052            raw_arguments: serde_json::to_string(&arguments)?,
1053            arguments,
1054            thread_id: thread_id.clone(),
1055            turn_id: "slash-command".to_string(),
1056        };
1057        let runtime_config = self.status().await;
1058        let ctx = self.tool_execution_context(
1059            thread_id,
1060            "slash-command".to_string(),
1061            runtime_config.policy_mode,
1062            runtime_config.workspace.as_deref(),
1063            Some(&runtime_config.command_shell),
1064        );
1065        executor.execute(ctx, tool_call).await
1066    }
1067
1068    pub(crate) fn tool_execution_context(
1069        &self,
1070        thread_id: ThreadId,
1071        turn_id: TurnId,
1072        mode: PolicyMode,
1073        workspace: Option<&str>,
1074        command_shell: Option<&str>,
1075    ) -> ToolExecutionContext {
1076        let mut ctx = ToolExecutionContext::new(thread_id, turn_id, mode)
1077            .with_command_shell(command_shell.unwrap_or_default())
1078            .with_process_runner(Arc::new(LocalProcessRunner))
1079            .with_context_artifacts(self.context_artifacts.backend())
1080            .with_goal_controller(self.goals.clone())
1081            .with_subagent_trace_sink(Arc::new(RuntimeSubagentTraceSink::new(
1082                self.bus.clone(),
1083                self.thread_store.clone(),
1084            )))
1085            .with_swarm_progress_sink(Arc::new(RuntimeAgentSwarmProgressSink::new(
1086                self.bus.clone(),
1087                self.thread_store.clone(),
1088            )));
1089        if let Some(workspace) = workspace {
1090            ctx = ctx.with_workspace_handle(Arc::new(ScopedFilesystem::new(workspace)));
1091        }
1092        ctx
1093    }
1094
1095    pub async fn status(&self) -> RuntimeConfig {
1096        self.config.read().await.clone()
1097    }
1098
1099    pub async fn set_skills(&self, skills: SkillRegistry) {
1100        *self.skills.write().await = skills;
1101    }
1102
1103    pub async fn skills_snapshot(&self) -> SkillRegistry {
1104        self.skills.read().await.clone()
1105    }
1106
1107    pub fn workspace(&self) -> PathBuf {
1108        self.workspace.clone()
1109    }
1110
1111    /// Controls whether native workspace tools may execute without an
1112    /// explicit remote-runner binding. Hosted runtimes disable this after
1113    /// construction; ordinary local runtimes retain the compatible default.
1114    pub fn set_allow_local_workspaces(&self, allowed: bool) {
1115        self.allow_local_workspaces
1116            .store(allowed, Ordering::Relaxed);
1117    }
1118
1119    pub fn allows_local_workspaces(&self) -> bool {
1120        self.allow_local_workspaces.load(Ordering::Relaxed)
1121    }
1122
1123    pub async fn set_remote_runner_destination(&self, destination: Option<RunnerDestination>) {
1124        let lifecycle = destination.as_ref().map(|destination| RunnerLifecycle {
1125            destination_id: destination.id.clone(),
1126            provider_id: destination.provider_id.clone(),
1127            state: "configured".to_string(),
1128            session_id: None,
1129            timestamp: OffsetDateTime::now_utc(),
1130        });
1131        self.config.write().await.remote_runner_destination = destination;
1132        if let Some(lifecycle) = lifecycle {
1133            self.emit(RoderEvent::RunnerLifecycle(lifecycle)).await;
1134        } else {
1135            self.emit(RoderEvent::RunnerLifecycle(RunnerLifecycle {
1136                destination_id: "local".to_string(),
1137                provider_id: "local".to_string(),
1138                state: "local_fallback".to_string(),
1139                session_id: None,
1140                timestamp: OffsetDateTime::now_utc(),
1141            }))
1142            .await;
1143        }
1144    }
1145
1146    pub async fn set_file_backed_dynamic_context(&self, enabled: bool) -> RuntimeConfig {
1147        let mut cfg = self.config.write().await;
1148        cfg.file_backed_dynamic_context = enabled;
1149        cfg.clone()
1150    }
1151
1152    pub async fn set_command_shell(&self, shell: String) -> RuntimeConfig {
1153        let mut cfg = self.config.write().await;
1154        cfg.command_shell = shell;
1155        cfg.clone()
1156    }
1157
1158    pub async fn pending_plan_exit(&self) -> Option<PendingPlanExit> {
1159        let mut pending = self.pending_plan_exit.write().await;
1160        let current = pending.clone()?;
1161        if !current.is_expired(OffsetDateTime::now_utc()) {
1162            return Some(current);
1163        }
1164        *pending = None;
1165        drop(pending);
1166        self.emit_plan_exit_resolved(&current, false, self.status().await.policy_mode)
1167            .await;
1168        None
1169    }
1170
1171    pub async fn set_policy_mode(
1172        &self,
1173        mode: PolicyMode,
1174        reason: Option<String>,
1175    ) -> anyhow::Result<RuntimeConfig> {
1176        let mut cfg = self.config.write().await;
1177        let previous_mode = cfg.policy_mode;
1178        cfg.policy_mode = mode;
1179        let next = cfg.clone();
1180        drop(cfg);
1181        self.emit(RoderEvent::PolicyModeChanged(PolicyModeChanged {
1182            thread_id: "runtime".to_string(),
1183            turn_id: None,
1184            previous_mode,
1185            new_mode: mode,
1186            reason,
1187            timestamp: OffsetDateTime::now_utc(),
1188        }))
1189        .await;
1190        self.auto_resolve_pending_tool_approvals_for_mode(mode)
1191            .await;
1192        Ok(next)
1193    }
1194
1195    /// Toggle agent-swarm mode for the runtime (roadmap 104). When enabled, the
1196    /// swarm reminder is injected into each turn's developer instructions so
1197    /// every app-server/SDK client gets it, not only the TUI.
1198    pub async fn set_agent_swarm_mode(
1199        &self,
1200        enabled: bool,
1201        trigger: roder_api::subagents::AgentSwarmModeTrigger,
1202    ) -> anyhow::Result<RuntimeConfig> {
1203        let mut cfg = self.config.write().await;
1204        cfg.agent_swarm_mode = enabled;
1205        let next = cfg.clone();
1206        drop(cfg);
1207        self.emit(RoderEvent::AgentSwarmModeChanged(
1208            roder_api::subagents::AgentSwarmModeChanged {
1209                thread_id: "runtime".to_string(),
1210                turn_id: None,
1211                enabled,
1212                trigger,
1213                timestamp: OffsetDateTime::now_utc(),
1214            },
1215        ))
1216        .await;
1217        Ok(next)
1218    }
1219
1220    /// Toggle agent-swarm mode for a single thread (roadmap 104). The override is
1221    /// stored per thread so toggling swarm mode on one thread does not leak the
1222    /// reminder into other threads sharing the runtime; absent threads fall back
1223    /// to the runtime-global default. The emitted `AgentSwarmModeChanged` event
1224    /// carries the real `thread_id`.
1225    pub async fn set_agent_swarm_mode_for_thread(
1226        &self,
1227        thread_id: &str,
1228        enabled: bool,
1229        trigger: roder_api::subagents::AgentSwarmModeTrigger,
1230    ) -> bool {
1231        {
1232            let mut modes = self.agent_swarm_modes.write().await;
1233            modes.insert(thread_id.to_string(), enabled);
1234        }
1235        self.emit(RoderEvent::AgentSwarmModeChanged(
1236            roder_api::subagents::AgentSwarmModeChanged {
1237                thread_id: thread_id.to_string(),
1238                turn_id: None,
1239                enabled,
1240                trigger,
1241                timestamp: OffsetDateTime::now_utc(),
1242            },
1243        ))
1244        .await;
1245        enabled
1246    }
1247
1248    /// Resolve whether agent-swarm mode is active for `thread_id`: the per-thread
1249    /// override when present, otherwise the runtime-global default. The turn loop
1250    /// uses this to decide whether to inject the swarm reminder.
1251    pub async fn effective_agent_swarm_mode_for_thread(&self, thread_id: &str) -> bool {
1252        if let Some(enabled) = self.agent_swarm_modes.read().await.get(thread_id).copied() {
1253            return enabled;
1254        }
1255        self.status().await.agent_swarm_mode
1256    }
1257
1258    /// Toggle ultra mode for the runtime. When enabled, proactive multi-agent
1259    /// policy is injected into each turn's developer instructions for any
1260    /// model, so every app-server/SDK client gets it (not only the TUI).
1261    pub async fn set_ultra_mode(
1262        &self,
1263        enabled: bool,
1264        trigger: roder_api::subagents::UltraModeTrigger,
1265    ) -> anyhow::Result<RuntimeConfig> {
1266        let mut cfg = self.config.write().await;
1267        cfg.ultra_mode = enabled;
1268        let next = cfg.clone();
1269        drop(cfg);
1270        self.emit(RoderEvent::UltraModeChanged(
1271            roder_api::subagents::UltraModeChanged {
1272                thread_id: "runtime".to_string(),
1273                turn_id: None,
1274                enabled,
1275                trigger,
1276                timestamp: OffsetDateTime::now_utc(),
1277            },
1278        ))
1279        .await;
1280        Ok(next)
1281    }
1282
1283    /// Toggle ultra mode for a single thread. The override is stored per thread
1284    /// so toggling on one thread does not leak proactive multi-agent policy into
1285    /// other threads sharing the runtime; absent threads fall back to the
1286    /// runtime-global default.
1287    pub async fn set_ultra_mode_for_thread(
1288        &self,
1289        thread_id: &str,
1290        enabled: bool,
1291        trigger: roder_api::subagents::UltraModeTrigger,
1292    ) -> bool {
1293        {
1294            let mut modes = self.ultra_modes.write().await;
1295            modes.insert(thread_id.to_string(), enabled);
1296        }
1297        self.emit(RoderEvent::UltraModeChanged(
1298            roder_api::subagents::UltraModeChanged {
1299                thread_id: thread_id.to_string(),
1300                turn_id: None,
1301                enabled,
1302                trigger,
1303                timestamp: OffsetDateTime::now_utc(),
1304            },
1305        ))
1306        .await;
1307        enabled
1308    }
1309
1310    /// Resolve whether ultra mode is active for `thread_id`: the per-thread
1311    /// override when present, otherwise the runtime-global default. The turn
1312    /// loop uses this (together with Sol/Terra Ultra effort) to decide whether
1313    /// to inject proactive multi-agent policy.
1314    pub async fn effective_ultra_mode_for_thread(&self, thread_id: &str) -> bool {
1315        if let Some(enabled) = self.ultra_modes.read().await.get(thread_id).copied() {
1316            return enabled;
1317        }
1318        self.status().await.ultra_mode
1319    }
1320
1321    pub async fn set_hosted_web_search(
1322        &self,
1323        mode: HostedWebSearchMode,
1324    ) -> anyhow::Result<RuntimeConfig> {
1325        let mut cfg = self.config.write().await;
1326        cfg.hosted_web_search = HostedWebSearchConfig { mode };
1327        Ok(cfg.clone())
1328    }
1329
1330    async fn auto_resolve_pending_tool_approvals_for_mode(&self, mode: PolicyMode) {
1331        let gate = DefaultPolicyGate::new();
1332        let mut pending = self.pending_tool_approvals.lock().await;
1333        let approval_ids = pending
1334            .iter()
1335            .filter_map(|(approval_id, approval)| {
1336                let ctx = ToolExecutionContext::new(
1337                    approval.thread_id.clone(),
1338                    approval.turn_id.clone(),
1339                    mode,
1340                );
1341                matches!(
1342                    gate.decide(&approval.call, mode, &ctx),
1343                    PolicyDecision::AutoApproved { .. }
1344                )
1345                .then_some(approval_id.clone())
1346            })
1347            .collect::<Vec<_>>();
1348        let approvals = approval_ids
1349            .into_iter()
1350            .filter_map(|approval_id| {
1351                pending
1352                    .remove(&approval_id)
1353                    .map(|approval| (approval_id, approval))
1354            })
1355            .collect::<Vec<_>>();
1356        drop(pending);
1357
1358        for (approval_id, approval) in approvals {
1359            let ctx = ToolExecutionContext::new(
1360                approval.thread_id.clone(),
1361                approval.turn_id.clone(),
1362                mode,
1363            );
1364            let decision = gate.decide(&approval.call, mode, &ctx);
1365            self.emit(RoderEvent::PolicyDecisionRecorded(PolicyDecisionRecorded {
1366                thread_id: approval.thread_id.clone(),
1367                turn_id: approval.turn_id.clone(),
1368                tool_id: approval.tool_id.clone(),
1369                tool_name: approval.tool_name.clone(),
1370                mode,
1371                decision,
1372                timestamp: OffsetDateTime::now_utc(),
1373            }))
1374            .await;
1375            if mode == PolicyMode::Bypass {
1376                self.emit(RoderEvent::PolicyBypassActive(PolicyBypassActive {
1377                    thread_id: approval.thread_id.clone(),
1378                    turn_id: approval.turn_id.clone(),
1379                    tool_id: approval.tool_id.clone(),
1380                    tool_name: approval.tool_name.clone(),
1381                    timestamp: OffsetDateTime::now_utc(),
1382                }))
1383                .await;
1384            }
1385            self.emit(RoderEvent::ApprovalResolved(ApprovalResolved {
1386                thread_id: approval.thread_id,
1387                turn_id: approval.turn_id,
1388                approval_id,
1389                tool_id: approval.tool_id,
1390                tool_name: approval.tool_name,
1391                approved: true,
1392                timestamp: OffsetDateTime::now_utc(),
1393            }))
1394            .await;
1395            let _ = approval.tx.send(true);
1396        }
1397    }
1398
1399    pub async fn record_pending_plan_exit(&self, pending: PendingPlanExit) {
1400        *self.pending_plan_exit.write().await = Some(pending.clone());
1401        self.emit(RoderEvent::PolicyExitPlanRequested(
1402            PolicyExitPlanRequested {
1403                thread_id: pending.thread_id,
1404                turn_id: pending.turn_id,
1405                request_id: pending.request_id,
1406                target_mode: pending.target_mode,
1407                plan_summary: pending.plan_summary,
1408                next_steps: pending.next_steps,
1409                timestamp: OffsetDateTime::now_utc(),
1410            },
1411        ))
1412        .await;
1413    }
1414
1415    pub async fn resolve_pending_plan_exit(
1416        &self,
1417        request_id: &str,
1418        approved: bool,
1419    ) -> anyhow::Result<Option<PendingPlanExit>> {
1420        let mut pending = self.pending_plan_exit.write().await;
1421        let Some(current) = pending.clone() else {
1422            return Ok(None);
1423        };
1424        if current.request_id != request_id {
1425            anyhow::bail!("pending plan exit request {request_id:?} was not found");
1426        }
1427        *pending = None;
1428        drop(pending);
1429
1430        let approved = approved && !current.is_expired(OffsetDateTime::now_utc());
1431        let resolved_mode = if approved {
1432            let mut cfg = self.config.write().await;
1433            let previous_mode = cfg.policy_mode;
1434            cfg.policy_mode = current.target_mode;
1435            drop(cfg);
1436            self.emit(RoderEvent::PolicyModeChanged(PolicyModeChanged {
1437                thread_id: current.thread_id.clone(),
1438                turn_id: Some(current.turn_id.clone()),
1439                previous_mode,
1440                new_mode: current.target_mode,
1441                reason: Some("approved plan exit".to_string()),
1442                timestamp: OffsetDateTime::now_utc(),
1443            }))
1444            .await;
1445            self.auto_resolve_pending_tool_approvals_for_mode(current.target_mode)
1446                .await;
1447            current.target_mode
1448        } else {
1449            self.status().await.policy_mode
1450        };
1451        self.emit_plan_exit_resolved(&current, approved, resolved_mode)
1452            .await;
1453        Ok(Some(current))
1454    }
1455
1456    pub async fn resolve_tool_approval(
1457        &self,
1458        approval_id: &str,
1459        approved: bool,
1460    ) -> anyhow::Result<bool> {
1461        let pending = self.pending_tool_approvals.lock().await.remove(approval_id);
1462        let Some(pending) = pending else {
1463            return Ok(false);
1464        };
1465        self.emit(RoderEvent::ApprovalResolved(ApprovalResolved {
1466            thread_id: pending.thread_id,
1467            turn_id: pending.turn_id,
1468            approval_id: approval_id.to_string(),
1469            tool_id: pending.tool_id,
1470            tool_name: pending.tool_name,
1471            approved,
1472            timestamp: OffsetDateTime::now_utc(),
1473        }))
1474        .await;
1475        let _ = pending.tx.send(approved);
1476        Ok(true)
1477    }
1478
1479    pub async fn request_app_server_tool_approval(
1480        &self,
1481        call: ToolCall,
1482        reason: Option<String>,
1483    ) -> anyhow::Result<bool> {
1484        let approval_id = call.id.clone();
1485        let (tx, rx) = oneshot::channel();
1486        self.pending_tool_approvals.lock().await.insert(
1487            approval_id.clone(),
1488            PendingToolApproval {
1489                thread_id: call.thread_id.clone(),
1490                turn_id: call.turn_id.clone(),
1491                tool_id: call.id.clone(),
1492                tool_name: call.name.clone(),
1493                call: call.clone(),
1494                tx,
1495            },
1496        );
1497        self.emit(RoderEvent::ApprovalRequested(ApprovalRequested {
1498            thread_id: call.thread_id.clone(),
1499            turn_id: call.turn_id.clone(),
1500            approval_id,
1501            tool_id: call.id.clone(),
1502            tool_name: call.name.clone(),
1503            reason,
1504            timestamp: OffsetDateTime::now_utc(),
1505        }))
1506        .await;
1507        Ok(rx.await.unwrap_or(false))
1508    }
1509
1510    /// Completes a pending host-executed tool call (`tools/resolve`). Returns false when the
1511    /// request id is unknown, already resolved, timed out, or cancelled by a turn interrupt.
1512    pub async fn resolve_external_tool_call(
1513        &self,
1514        request_id: &str,
1515        resolution: ExternalToolResolution,
1516    ) -> anyhow::Result<bool> {
1517        let pending = self
1518            .pending_external_tool_calls
1519            .lock()
1520            .await
1521            .remove(request_id);
1522        let Some(pending) = pending else {
1523            return Ok(false);
1524        };
1525        self.emit(RoderEvent::ExternalToolCallResolved(
1526            ExternalToolCallResolved {
1527                thread_id: pending.thread_id,
1528                turn_id: pending.turn_id,
1529                request_id: request_id.to_string(),
1530                tool_id: pending.tool_id,
1531                tool_name: pending.tool_name,
1532                outcome: ExternalToolCallOutcome::Resolved,
1533                is_error: resolution.is_error,
1534                timestamp: OffsetDateTime::now_utc(),
1535            },
1536        ))
1537        .await;
1538        let _ = pending.tx.send(resolution);
1539        Ok(true)
1540    }
1541
1542    /// Drops every pending external tool call for the turn and reports them as cancelled so an
1543    /// interrupt does not leak requests waiting on `tools/resolve`.
1544    async fn cancel_pending_external_tool_calls_for_turn(&self, turn_id: &TurnId) {
1545        let cancelled = {
1546            let mut pending = self.pending_external_tool_calls.lock().await;
1547            let request_ids = pending
1548                .iter()
1549                .filter(|(_, call)| &call.turn_id == turn_id)
1550                .map(|(request_id, _)| request_id.clone())
1551                .collect::<Vec<_>>();
1552            request_ids
1553                .into_iter()
1554                .filter_map(|request_id| pending.remove(&request_id).map(|call| (request_id, call)))
1555                .collect::<Vec<_>>()
1556        };
1557        for (request_id, call) in cancelled {
1558            self.emit(RoderEvent::ExternalToolCallResolved(
1559                ExternalToolCallResolved {
1560                    thread_id: call.thread_id,
1561                    turn_id: call.turn_id,
1562                    request_id,
1563                    tool_id: call.tool_id,
1564                    tool_name: call.tool_name,
1565                    outcome: ExternalToolCallOutcome::Cancelled,
1566                    is_error: true,
1567                    timestamp: OffsetDateTime::now_utc(),
1568                },
1569            ))
1570            .await;
1571        }
1572    }
1573
1574    pub async fn resolve_user_input(
1575        &self,
1576        request_id: &str,
1577        answers: serde_json::Value,
1578    ) -> anyhow::Result<bool> {
1579        let pending = self.pending_user_inputs.lock().await.remove(request_id);
1580        let Some(pending) = pending else {
1581            return Ok(false);
1582        };
1583        self.emit(RoderEvent::UserInputResolved(UserInputResolved {
1584            thread_id: pending.thread_id,
1585            turn_id: pending.turn_id,
1586            request_id: request_id.to_string(),
1587            answers: answers.clone(),
1588            timestamp: OffsetDateTime::now_utc(),
1589        }))
1590        .await;
1591        let _ = pending.tx.send(answers);
1592        Ok(true)
1593    }
1594
1595    async fn emit_plan_exit_resolved(
1596        &self,
1597        current: &PendingPlanExit,
1598        approved: bool,
1599        resolved_mode: PolicyMode,
1600    ) {
1601        self.emit(RoderEvent::PolicyExitPlanResolved(PolicyExitPlanResolved {
1602            thread_id: current.thread_id.clone(),
1603            turn_id: current.turn_id.clone(),
1604            request_id: current.request_id.clone(),
1605            approved,
1606            target_mode: current.target_mode,
1607            resolved_mode,
1608            timestamp: OffsetDateTime::now_utc(),
1609        }))
1610        .await;
1611    }
1612
1613    pub async fn select_provider(
1614        &self,
1615        provider: String,
1616        model: Option<String>,
1617        reasoning: Option<String>,
1618    ) -> anyhow::Result<RuntimeConfig> {
1619        let next = self
1620            .preview_provider_selection(provider, model, reasoning)
1621            .await?;
1622        let mut cfg = self.config.write().await;
1623        *cfg = next;
1624        Ok(cfg.clone())
1625    }
1626
1627    pub async fn preview_provider_selection(
1628        &self,
1629        provider: String,
1630        model: Option<String>,
1631        reasoning: Option<String>,
1632    ) -> anyhow::Result<RuntimeConfig> {
1633        self.engine_for(&provider)?;
1634        let mut cfg = self.config.read().await.clone();
1635        cfg.default_provider = provider;
1636        if let Some(model) = model {
1637            cfg.default_model = model;
1638        }
1639        if let Some(reasoning) = reasoning {
1640            if reasoning == REASONING_NONE
1641                && !model_supports_reasoning(&cfg.default_model, &reasoning)
1642            {
1643                return Ok(cfg.clone());
1644            }
1645            validate_reasoning_effort(&cfg.default_model, &reasoning)?;
1646            cfg.reasoning = Some(reasoning);
1647        }
1648        Ok(cfg)
1649    }
1650
1651    pub async fn effective_reasoning(&self) -> String {
1652        let cfg = self.config.read().await;
1653        effective_reasoning_for_model(&cfg, &cfg.default_model)
1654    }
1655
1656    pub fn effective_reasoning_for_config(cfg: &RuntimeConfig) -> String {
1657        effective_reasoning_for_model(cfg, &cfg.default_model)
1658    }
1659
1660    pub async fn set_dynamic_workflow_effort(
1661        &self,
1662        effort_profile: DynamicWorkflowEffortProfile,
1663    ) -> RuntimeConfig {
1664        let mut cfg = self.config.write().await;
1665        cfg.dynamic_workflows.effort_profile = effort_profile;
1666        cfg.clone()
1667    }
1668
1669    pub async fn dynamic_workflow_trigger_decision(
1670        &self,
1671        message: &str,
1672    ) -> WorkflowTriggerDecision {
1673        let cfg = self.config.read().await;
1674        classify_workflow_trigger(message, &cfg.dynamic_workflows)
1675    }
1676
1677    pub async fn create_thread(&self, title: Option<String>) -> anyhow::Result<ThreadMetadata> {
1678        self.create_thread_with(CreateThreadRequest {
1679            title,
1680            workspace: self.workspace.display().to_string(),
1681            workspace_id: None,
1682            root_id: None,
1683            provider: None,
1684            model: None,
1685            selection_mode: None,
1686            tool_allowlist: Vec::new(),
1687            developer_instructions: None,
1688            external_tools: Vec::new(),
1689            runner: None,
1690        })
1691        .await
1692    }
1693
1694    /**
1695     * Resolves an explicit thread-runner selection into a persisted binding.
1696     * Fails fast at thread creation when the provider is missing or rejects
1697     * the destination, instead of surfacing the error on the first tool call.
1698     */
1699    async fn resolve_thread_runner_binding(
1700        &self,
1701        thread_id: &str,
1702        selection: ThreadRunnerSelection,
1703    ) -> anyhow::Result<ThreadRunnerBinding> {
1704        let provider = self
1705            .registry
1706            .remote_runner_providers
1707            .iter()
1708            .find(|provider| provider.id() == selection.provider_id)
1709            .cloned()
1710            .ok_or_else(|| {
1711                anyhow::anyhow!(
1712                    "remote runner provider {:?} is not installed",
1713                    selection.provider_id
1714                )
1715            })?;
1716        let workspace = selection.workspace.trim();
1717        anyhow::ensure!(
1718            std::path::Path::new(workspace).is_absolute(),
1719            "runner workspace must be an absolute path on the runner: {workspace:?}"
1720        );
1721        let mut read_roots = Vec::with_capacity(selection.read_roots.len());
1722        for read_root in &selection.read_roots {
1723            let trimmed = read_root.trim();
1724            anyhow::ensure!(
1725                std::path::Path::new(trimmed).is_absolute(),
1726                "runner read root must be an absolute path on the runner: {trimmed:?}"
1727            );
1728            read_roots.push(PathBuf::from(trimmed));
1729        }
1730        let destination = RunnerDestination {
1731            id: format!("thread-{thread_id}"),
1732            provider_id: selection.provider_id,
1733            config: selection.config,
1734            default_manifest: roder_api::remote_runner::RunnerManifest::default(),
1735        };
1736        provider.validate_destination(&destination).await?;
1737        Ok(ThreadRunnerBinding {
1738            destination,
1739            workspace: PathBuf::from(workspace),
1740            read_roots,
1741        })
1742    }
1743
1744    /**
1745     * Build a per-thread binding from a runtime-level destination (selected via
1746     * the TUI runner picker or config `default_destination`) when the
1747     * destination's provider advertises a default workspace. Returns `None` when
1748     * there is no destination or the provider opts out (no default workspace),
1749     * preserving the legacy behavior where such threads keep local tools.
1750     */
1751    async fn synthesize_runtime_runner_binding(
1752        &self,
1753        thread_id: &str,
1754        destination: Option<RunnerDestination>,
1755    ) -> anyhow::Result<Option<ThreadRunnerBinding>> {
1756        let Some(destination) = destination else {
1757            return Ok(None);
1758        };
1759        let Some(provider) = self
1760            .registry
1761            .remote_runner_providers
1762            .iter()
1763            .find(|provider| provider.id() == destination.provider_id)
1764        else {
1765            return Ok(None);
1766        };
1767        // An explicit workspace in the destination config wins over the
1768        // provider default; absence of both means the provider opts out.
1769        let workspace = destination
1770            .config
1771            .get("working_dir")
1772            .and_then(serde_json::Value::as_str)
1773            .map(str::to_string)
1774            .or_else(|| provider.default_workspace());
1775        let Some(workspace) = workspace else {
1776            return Ok(None);
1777        };
1778        let selection = ThreadRunnerSelection {
1779            provider_id: destination.provider_id.clone(),
1780            config: destination.config.clone(),
1781            workspace,
1782            read_roots: Vec::new(),
1783        };
1784        Ok(Some(
1785            self.resolve_thread_runner_binding(thread_id, selection)
1786                .await?,
1787        ))
1788    }
1789
1790    /// Validates a runner selection without creating a thread; the placeholder destination id only appears in error messages.
1791    pub async fn validate_thread_runner_selection(
1792        &self,
1793        selection: ThreadRunnerSelection,
1794    ) -> anyhow::Result<()> {
1795        self.resolve_thread_runner_binding("validate", selection)
1796            .await
1797            .map(|_| ())
1798    }
1799
1800    pub async fn create_thread_with(
1801        &self,
1802        req: CreateThreadRequest,
1803    ) -> anyhow::Result<ThreadMetadata> {
1804        tool_advertisement::validate_external_tool_names(&req.external_tools)?;
1805        let cfg = self.config.read().await.clone();
1806        let now = OffsetDateTime::now_utc();
1807        let workspace = validate_thread_workspace(&req.workspace)?;
1808        let provider = req.provider.unwrap_or(cfg.default_provider);
1809        let model = req.model.unwrap_or(cfg.default_model);
1810        let selection_mode = req
1811            .selection_mode
1812            .unwrap_or_else(|| ModelSelectionMode::manual(provider.clone(), model.clone(), None));
1813        let thread_id = uuid::Uuid::new_v4().to_string();
1814        let runner_binding = match req.runner {
1815            Some(selection) => Some(
1816                self.resolve_thread_runner_binding(&thread_id, selection)
1817                    .await?,
1818            ),
1819            // No explicit per-thread selection: if a runtime-level destination
1820            // is active and its provider advertises a default workspace, bind
1821            // the new thread so its coding tools route into the runner. This is
1822            // what makes a TUI/config-selected runner (e.g. Blaxel) actually
1823            // execute tools in the sandbox instead of locally. Providers that
1824            // return no default workspace stay unbound (legacy local-tools).
1825            None => {
1826                self.synthesize_runtime_runner_binding(
1827                    &thread_id,
1828                    cfg.remote_runner_destination.clone(),
1829                )
1830                .await?
1831            }
1832        };
1833        let runner_destination = runner_binding
1834            .as_ref()
1835            .map(|binding| binding.destination.clone())
1836            .or_else(|| cfg.remote_runner_destination.clone());
1837        let metadata = ThreadMetadata {
1838            thread_id,
1839            title: req.title,
1840            workspace,
1841            workspace_id: req.workspace_id,
1842            root_id: req.root_id,
1843            provider: Some(provider),
1844            model: Some(model),
1845            selection_mode: Some(selection_mode),
1846            tool_allowlist: req.tool_allowlist,
1847            developer_instructions: req.developer_instructions,
1848            external_tools: req.external_tools,
1849            runner_destination,
1850            runner_state: None,
1851            runner_binding,
1852            created_at: now,
1853            updated_at: now,
1854            message_count: 0,
1855            usage: None,
1856            parent_thread_id: None,
1857            forked_from_turn_id: None,
1858            workspace_fork: None,
1859        };
1860
1861        let metadata = if let Some(store) = &self.thread_store {
1862            store.create_thread(metadata).await?
1863        } else {
1864            metadata
1865        };
1866        self.emit(RoderEvent::ThreadCreated(ThreadCreated {
1867            thread_id: metadata.thread_id.clone(),
1868            timestamp: OffsetDateTime::now_utc(),
1869        }))
1870        .await;
1871        Ok(metadata)
1872    }
1873
1874    pub async fn list_threads(&self) -> anyhow::Result<Vec<ThreadMetadata>> {
1875        if let Some(store) = &self.thread_store {
1876            return store.list_threads().await;
1877        }
1878        Ok(Vec::new())
1879    }
1880
1881    pub async fn list_threads_page(
1882        &self,
1883        options: roder_api::thread::ThreadListOptions,
1884    ) -> anyhow::Result<roder_api::thread::ThreadListPage> {
1885        if let Some(store) = &self.thread_store {
1886            return store.list_threads_page(options).await;
1887        }
1888        Ok(roder_api::thread::ThreadListPage::default())
1889    }
1890
1891    pub async fn load_thread_metadata(
1892        &self,
1893        thread_id: &str,
1894    ) -> anyhow::Result<Option<roder_api::thread::ThreadMetadata>> {
1895        if let Some(store) = &self.thread_store {
1896            return store.load_thread_metadata(&thread_id.to_string()).await;
1897        }
1898        Ok(None)
1899    }
1900
1901    pub async fn archive_thread(&self, thread_id: &str) -> anyhow::Result<bool> {
1902        let archived = if let Some(store) = &self.thread_store {
1903            store.archive_thread(&thread_id.to_string()).await?
1904        } else {
1905            false
1906        };
1907        if archived {
1908            let workspace = self.config.read().await.workspace.clone();
1909            crate::hooks::run_lifecycle(
1910                self,
1911                &thread_id.to_string(),
1912                &SESSION_HOOK_TURN_ID.to_string(),
1913                workspace.as_deref(),
1914                "SessionEnd",
1915                Some("clear"),
1916                serde_json::json!({"reason": "archive"}),
1917            )
1918            .await;
1919            self.thread_item_cache
1920                .lock()
1921                .await
1922                .remove_thread(&thread_id.to_string());
1923            self.evict_runner_session(&thread_id.to_string()).await;
1924        }
1925        Ok(archived)
1926    }
1927
1928    pub async fn start_team(&self, req: TeamStartRequest) -> anyhow::Result<TeamState> {
1929        let cfg = self.config.read().await.clone();
1930        let workspace = self.workspace.display().to_string();
1931        let lead_thread_id = match req.lead_thread_id {
1932            Some(thread_id) => thread_id,
1933            None => {
1934                self.create_thread_with(CreateThreadRequest {
1935                    title: Some("Team lead".to_string()),
1936                    workspace: workspace.clone(),
1937                    workspace_id: None,
1938                    root_id: None,
1939                    provider: None,
1940                    model: None,
1941                    selection_mode: None,
1942                    tool_allowlist: Vec::new(),
1943                    developer_instructions: None,
1944                    external_tools: Vec::new(),
1945                    runner: None,
1946                })
1947                .await?
1948                .thread_id
1949            }
1950        };
1951        let team_id = uuid::Uuid::new_v4().to_string();
1952        let active_lead_turn_id = self.active_turn_for_thread(&lead_thread_id).await;
1953        let lead_selection = match active_lead_turn_id.as_ref() {
1954            Some(turn_id) => self
1955                .active_turn_selections
1956                .read()
1957                .await
1958                .get(turn_id)
1959                .cloned(),
1960            None => self.selection_mode_for_thread(&lead_thread_id).await?,
1961        };
1962        let lead_concrete_selection = lead_selection
1963            .as_ref()
1964            .map(ModelSelectionMode::concrete_selection);
1965        let mut lead = crate::teams::lead_member(
1966            lead_thread_id.clone(),
1967            lead_concrete_selection
1968                .as_ref()
1969                .map(|selection| selection.provider.clone())
1970                .or_else(|| Some(cfg.default_provider.clone())),
1971            lead_concrete_selection
1972                .as_ref()
1973                .map(|selection| selection.model.clone())
1974                .or_else(|| Some(cfg.default_model.clone())),
1975            cfg.policy_mode,
1976        );
1977        if let Some(turn_id) = active_lead_turn_id {
1978            lead.current_turn_id = Some(turn_id);
1979            lead.status = TeamMemberStatus::Running;
1980        }
1981        let mut members = vec![lead];
1982
1983        for (index, member) in req.members.into_iter().enumerate() {
1984            let thread = self
1985                .create_thread_with(CreateThreadRequest {
1986                    title: Some(member.name.clone()),
1987                    workspace: workspace.clone(),
1988                    workspace_id: None,
1989                    root_id: None,
1990                    provider: member.model_provider.clone(),
1991                    model: member.model.clone(),
1992                    selection_mode: None,
1993                    tool_allowlist: Vec::new(),
1994                    developer_instructions: None,
1995                    external_tools: Vec::new(),
1996                    runner: None,
1997                })
1998                .await?;
1999            let member_id = format!("member-{}", index + 1);
2000            let descriptor = crate::teams::teammate_member(
2001                member_id.clone(),
2002                member.name,
2003                thread.thread_id.clone(),
2004                member.model_provider.or(thread.provider),
2005                member.model.or(thread.model),
2006                cfg.policy_mode,
2007            );
2008            members.push(descriptor);
2009        }
2010
2011        let now = OffsetDateTime::now_utc();
2012        let team = self
2013            .teams
2014            .insert(TeamState {
2015                id: team_id.clone(),
2016                lead_thread_id: lead_thread_id.clone(),
2017                display_mode: req.display_mode,
2018                members,
2019                mailbox: Vec::new(),
2020                tasks: Vec::new(),
2021                created_at: now,
2022                updated_at: now,
2023            })
2024            .await?;
2025        self.emit(RoderEvent::TeamStarted(TeamStarted {
2026            team_id: team_id.clone(),
2027            lead_thread_id,
2028            display_mode: team.display_mode,
2029            members: team.members.clone(),
2030            tasks: team.tasks.clone(),
2031            timestamp: OffsetDateTime::now_utc(),
2032        }))
2033        .await;
2034        for member in team
2035            .members
2036            .iter()
2037            .filter(|member| member.role != roder_api::teams::TeamMemberRole::Lead)
2038        {
2039            self.emit(RoderEvent::TeamMemberStarted(TeamMemberStarted {
2040                team_id: team_id.clone(),
2041                member: member.clone(),
2042                timestamp: OffsetDateTime::now_utc(),
2043            }))
2044            .await;
2045        }
2046        Ok(team)
2047    }
2048
2049    pub async fn list_teams(&self) -> Vec<TeamState> {
2050        let active_thread_ids = self
2051            .active_turns
2052            .read()
2053            .await
2054            .values()
2055            .map(|handle| handle.thread_id.clone())
2056            .collect::<std::collections::HashSet<_>>();
2057        self.teams
2058            .list()
2059            .await
2060            .into_iter()
2061            .map(|team| runtime_local_team_view(team, &active_thread_ids))
2062            .collect()
2063    }
2064
2065    pub async fn read_team(&self, team_id: &str) -> Option<TeamState> {
2066        let active_thread_ids = self
2067            .active_turns
2068            .read()
2069            .await
2070            .values()
2071            .map(|handle| handle.thread_id.clone())
2072            .collect::<std::collections::HashSet<_>>();
2073        self.teams
2074            .get(team_id)
2075            .await
2076            .map(|team| runtime_local_team_view(team, &active_thread_ids))
2077    }
2078
2079    pub async fn start_team_member(
2080        &self,
2081        team_id: &str,
2082        req: TeamMemberStartRequest,
2083    ) -> anyhow::Result<TeamState> {
2084        self.start_team_member_with_selection(team_id, req, None)
2085            .await
2086    }
2087
2088    pub async fn message_team_member(
2089        self: &Arc<Self>,
2090        team_id: &str,
2091        member_id: &str,
2092        message: String,
2093    ) -> anyhow::Result<TurnId> {
2094        let team = self
2095            .read_team(team_id)
2096            .await
2097            .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2098        self.followup_team_member(&team.lead_thread_id, team_id, member_id, message)
2099            .await
2100    }
2101
2102    /// Queue a mailbox message without starting an idle agent. If the target is
2103    /// already running, the message is delivered at the next inference boundary.
2104    pub(crate) async fn queue_team_member_message(
2105        self: &Arc<Self>,
2106        caller_thread_id: &ThreadId,
2107        team_id: &str,
2108        member_id: &str,
2109        message: String,
2110    ) -> anyhow::Result<Option<TurnId>> {
2111        let _delivery_guard = self.agent_team_spawn_lock.lock().await;
2112        let team = self
2113            .read_team(team_id)
2114            .await
2115            .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2116        let member = team
2117            .members
2118            .iter()
2119            .find(|member| member.id == member_id)
2120            .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2121            .clone();
2122        if member.status == TeamMemberStatus::Closed {
2123            anyhow::bail!("subagent {} is closed", member.name);
2124        }
2125        let from_member_id = team
2126            .members
2127            .iter()
2128            .find(|candidate| candidate.thread_id == *caller_thread_id)
2129            .map(|candidate| candidate.id.clone());
2130        self.teams
2131            .append_mailbox_message(
2132                team_id,
2133                from_member_id,
2134                member_id.to_string(),
2135                TeamMailboxMessageKind::Message,
2136                message,
2137            )
2138            .await?;
2139        let Some(turn_id) = self.active_turn_for_thread(&member.thread_id).await else {
2140            return Ok(None);
2141        };
2142        self.deliver_pending_team_mailbox(team_id, member_id, member.thread_id, turn_id.clone())
2143            .await?;
2144        Ok(Some(turn_id))
2145    }
2146
2147    /// Deliver queued messages and start an idle agent, or steer its active turn.
2148    pub(crate) async fn followup_team_member(
2149        self: &Arc<Self>,
2150        caller_thread_id: &ThreadId,
2151        team_id: &str,
2152        member_id: &str,
2153        message: String,
2154    ) -> anyhow::Result<TurnId> {
2155        let _spawn_guard = self.agent_team_spawn_lock.lock().await;
2156        self.followup_team_member_locked(caller_thread_id, team_id, member_id, message)
2157            .await
2158    }
2159
2160    async fn followup_team_member_locked(
2161        self: &Arc<Self>,
2162        caller_thread_id: &ThreadId,
2163        team_id: &str,
2164        member_id: &str,
2165        message: String,
2166    ) -> anyhow::Result<TurnId> {
2167        let team = self
2168            .read_team(team_id)
2169            .await
2170            .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2171        let member = team
2172            .members
2173            .iter()
2174            .find(|member| member.id == member_id)
2175            .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2176            .clone();
2177        if member.status == TeamMemberStatus::Closed {
2178            anyhow::bail!("subagent {} is closed", member.name);
2179        }
2180        let from_member_id = team
2181            .members
2182            .iter()
2183            .find(|candidate| candidate.thread_id == *caller_thread_id)
2184            .map(|candidate| candidate.id.clone());
2185        self.teams
2186            .append_mailbox_message(
2187                team_id,
2188                from_member_id,
2189                member_id.to_string(),
2190                TeamMailboxMessageKind::NewTask,
2191                message,
2192            )
2193            .await?;
2194        if let Some(turn_id) = self.active_turn_for_thread(&member.thread_id).await {
2195            self.deliver_pending_team_mailbox(
2196                team_id,
2197                member_id,
2198                member.thread_id,
2199                turn_id.clone(),
2200            )
2201            .await?;
2202            return Ok(turn_id);
2203        }
2204
2205        self.ensure_codex_v2_team_capacity(&team, &member.id, None)
2206            .await?;
2207        let metadata = self.load_thread_metadata(&member.thread_id).await?;
2208        let inherited_context = self
2209            .team_member_turn_contexts
2210            .lock()
2211            .await
2212            .get(&member.thread_id)
2213            .cloned();
2214        let workspace = inherited_context
2215            .as_ref()
2216            .map(|context| context.workspace.clone())
2217            .or_else(|| metadata.as_ref().map(|metadata| metadata.workspace.clone()))
2218            .unwrap_or_else(|| self.workspace.display().to_string());
2219        let member_thread_id = member.thread_id;
2220        let turn_id = self
2221            .start_turn(StartTurnRequest {
2222                thread_id: member_thread_id.clone(),
2223                message: String::new(),
2224                images: Vec::new(),
2225                provider_override: member.model_provider,
2226                model_override: member.model,
2227                reasoning_override: metadata
2228                    .as_ref()
2229                    .and_then(|metadata| metadata.selection_mode.as_ref())
2230                    .and_then(ModelSelectionMode::reasoning)
2231                    .map(str::to_string),
2232                workspace,
2233                instructions: inherited_context
2234                    .as_ref()
2235                    .map(|context| context.instructions.clone())
2236                    .unwrap_or_else(crate::default_instructions),
2237                developer_context: inherited_context
2238                    .as_ref()
2239                    .and_then(|context| context.developer_context.clone()),
2240                task_ledger_required: false,
2241                service_tier_override: None,
2242            })
2243            .await?;
2244        Ok(turn_id)
2245    }
2246
2247    pub async fn set_team_member_policy_mode(
2248        &self,
2249        team_id: &str,
2250        member_id: &str,
2251        policy_mode: PolicyMode,
2252    ) -> anyhow::Result<TeamState> {
2253        self.teams
2254            .set_member_policy_mode(team_id, member_id, policy_mode)
2255            .await
2256    }
2257
2258    pub async fn interrupt_team_member(
2259        &self,
2260        team_id: &str,
2261        member_id: &str,
2262    ) -> anyhow::Result<Option<TurnId>> {
2263        let _interrupt_guard = self.agent_team_spawn_lock.lock().await;
2264        let team = self
2265            .read_team(team_id)
2266            .await
2267            .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2268        let member = team
2269            .members
2270            .iter()
2271            .find(|member| member.id == member_id)
2272            .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2273            .clone();
2274        if member.status != TeamMemberStatus::Running {
2275            return Ok(None);
2276        }
2277        let Some(turn_id) = member.current_turn_id.clone() else {
2278            return Ok(None);
2279        };
2280        if self
2281            .active_turn_for_thread(&member.thread_id)
2282            .await
2283            .as_ref()
2284            != Some(&turn_id)
2285        {
2286            return Ok(None);
2287        }
2288        self.interrupt_turn(member.thread_id.clone(), turn_id.clone())
2289            .await?;
2290        // Make queued mailbox work immediately eligible for the replacement turn while the
2291        // lifecycle lock still prevents a follow-up from starting. Delivery acknowledgements
2292        // are turn-owned, so a late acknowledgement from the aborted turn cannot steal a
2293        // reservation acquired by its replacement.
2294        self.teams
2295            .release_mailbox_reservations_for_turn(&turn_id)
2296            .await;
2297        self.teams
2298            .update_member(team_id, member_id, |member| {
2299                member.status = TeamMemberStatus::Interrupted;
2300                member.current_turn_id = None;
2301                member.terminal_error = None;
2302            })
2303            .await?;
2304        self.emit(RoderEvent::TeamMemberCompleted(TeamMemberCompleted {
2305            team_id: team_id.to_string(),
2306            member_id: member_id.to_string(),
2307            member_thread_id: member.thread_id,
2308            turn_id: Some(turn_id.clone()),
2309            status: TeamMemberStatus::Interrupted,
2310            final_message: member.final_message,
2311            error: None,
2312            timestamp: OffsetDateTime::now_utc(),
2313        }))
2314        .await;
2315        Ok(Some(turn_id))
2316    }
2317
2318    pub async fn close_team_member(
2319        &self,
2320        team_id: &str,
2321        member_id: &str,
2322    ) -> anyhow::Result<roder_api::teams::TeamMemberDescriptor> {
2323        let team = self
2324            .read_team(team_id)
2325            .await
2326            .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2327        let member = team
2328            .members
2329            .iter()
2330            .find(|member| member.id == member_id)
2331            .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2332            .clone();
2333        if member.role == roder_api::teams::TeamMemberRole::Lead {
2334            anyhow::bail!("team lead cannot be closed as a subagent");
2335        }
2336        let interrupted_turn_id = if member.status == TeamMemberStatus::Running {
2337            if let Some(turn_id) = member.current_turn_id.clone() {
2338                self.interrupt_turn(member.thread_id.clone(), turn_id.clone())
2339                    .await?;
2340                Some(turn_id)
2341            } else {
2342                None
2343            }
2344        } else {
2345            member.current_turn_id.clone()
2346        };
2347        let updated = self
2348            .teams
2349            .update_member(team_id, member_id, |member| {
2350                member.status = TeamMemberStatus::Closed;
2351                member.current_turn_id = None;
2352            })
2353            .await?;
2354        let closed = updated
2355            .members
2356            .iter()
2357            .find(|member| member.id == member_id)
2358            .cloned()
2359            .ok_or_else(|| anyhow::anyhow!("closed team member disappeared"))?;
2360        self.team_member_turn_contexts
2361            .lock()
2362            .await
2363            .remove(&closed.thread_id);
2364        self.emit(RoderEvent::TeamMemberCompleted(TeamMemberCompleted {
2365            team_id: team_id.to_string(),
2366            member_id: closed.id.clone(),
2367            member_thread_id: closed.thread_id.clone(),
2368            turn_id: interrupted_turn_id,
2369            status: TeamMemberStatus::Closed,
2370            final_message: closed.final_message.clone(),
2371            error: closed.terminal_error.clone(),
2372            timestamp: OffsetDateTime::now_utc(),
2373        }))
2374        .await;
2375        Ok(closed)
2376    }
2377
2378    pub async fn cleanup_team(&self, team_id: &str, force: bool) -> anyhow::Result<bool> {
2379        let Some(team) = self.read_team(team_id).await else {
2380            return Ok(false);
2381        };
2382        if !force
2383            && team
2384                .members
2385                .iter()
2386                .any(|member| member.status == TeamMemberStatus::Running)
2387        {
2388            anyhow::bail!("team {team_id:?} has active teammates; use forced cleanup");
2389        }
2390        if force {
2391            for member in team.members.iter().filter(|member| {
2392                member.role != roder_api::teams::TeamMemberRole::Lead
2393                    && member.status == TeamMemberStatus::Running
2394            }) {
2395                if let Some(turn_id) = member
2396                    .current_turn_id
2397                    .clone()
2398                    .or(self.active_turn_for_thread(&member.thread_id).await)
2399                {
2400                    let _ = self.interrupt_turn(member.thread_id.clone(), turn_id).await;
2401                }
2402            }
2403        }
2404        let removed = self.teams.remove(team_id).await?.is_some();
2405        if removed {
2406            let member_thread_ids = team
2407                .members
2408                .iter()
2409                .map(|member| member.thread_id.clone())
2410                .collect::<std::collections::HashSet<_>>();
2411            self.team_member_turn_contexts
2412                .lock()
2413                .await
2414                .retain(|thread_id, _| !member_thread_ids.contains(thread_id));
2415            self.emit(RoderEvent::TeamCleanupCompleted(TeamCleanupCompleted {
2416                team_id: team_id.to_string(),
2417                forced: force,
2418                timestamp: OffsetDateTime::now_utc(),
2419            }))
2420            .await;
2421        }
2422        Ok(removed)
2423    }
2424
2425    pub async fn effective_policy_mode_for_thread(&self, thread_id: &str) -> PolicyMode {
2426        if let Some(mode) = self.teams.policy_mode_for_thread(thread_id).await {
2427            return mode;
2428        }
2429        self.status().await.policy_mode
2430    }
2431
2432    async fn complete_team_member_turn_with_result(
2433        &self,
2434        thread_id: &ThreadId,
2435        turn_id: &TurnId,
2436        status: TeamMemberStatus,
2437        final_message: Option<String>,
2438        terminal_error: Option<String>,
2439    ) -> anyhow::Result<()> {
2440        let _delivery_guard = self.agent_team_spawn_lock.lock().await;
2441        let Some((team_id, member)) = self
2442            .teams
2443            .complete_member_turn(
2444                thread_id,
2445                turn_id,
2446                status,
2447                final_message.clone(),
2448                terminal_error.clone(),
2449            )
2450            .await?
2451        else {
2452            return Ok(());
2453        };
2454        if let Some(parent_thread_id) = member.parent_thread_id.as_ref()
2455            && let Some(team) = self.read_team(&team_id).await
2456            && let Some(parent) = team
2457                .members
2458                .iter()
2459                .find(|candidate| candidate.thread_id == *parent_thread_id)
2460        {
2461            let identity = member
2462                .agent_path
2463                .as_deref()
2464                .or(member.task_name.as_deref())
2465                .unwrap_or(&member.name);
2466            let mut report = format!("Agent {identity} finished with status {status:?}.");
2467            if let Some(message) = final_message.as_deref()
2468                && !message.trim().is_empty()
2469            {
2470                report.push_str("\n\nFinal result:\n");
2471                report.push_str(message);
2472            }
2473            if let Some(error) = terminal_error.as_deref()
2474                && !error.trim().is_empty()
2475            {
2476                report.push_str("\n\nTerminal error:\n");
2477                report.push_str(error);
2478            }
2479            let mailbox_appended = self
2480                .teams
2481                .append_mailbox_message(
2482                    &team_id,
2483                    Some(member.id.clone()),
2484                    parent.id.clone(),
2485                    TeamMailboxMessageKind::FinalAnswer,
2486                    report,
2487                )
2488                .await
2489                .is_ok();
2490            if mailbox_appended
2491                && let Some(parent_turn_id) = self.active_turn_for_thread(parent_thread_id).await
2492            {
2493                // The parent may finish between the active-turn lookup and enqueue. Its durable
2494                // mailbox entry remains pending for the next turn, while terminal observers must
2495                // still receive TeamMemberCompleted for this child.
2496                let _ = self
2497                    .deliver_pending_team_mailbox(
2498                        &team_id,
2499                        &parent.id,
2500                        parent_thread_id.clone(),
2501                        parent_turn_id,
2502                    )
2503                    .await;
2504            }
2505        }
2506        self.emit(RoderEvent::TeamMemberCompleted(TeamMemberCompleted {
2507            team_id,
2508            member_id: member.id,
2509            member_thread_id: member.thread_id,
2510            turn_id: Some(turn_id.clone()),
2511            status,
2512            final_message,
2513            error: terminal_error,
2514            timestamp: OffsetDateTime::now_utc(),
2515        }))
2516        .await;
2517        Ok(())
2518    }
2519
2520    /// Stops accepting new turns, requests interruption for every active turn,
2521    /// and waits for their runtime tasks to reach terminal cleanup up to
2522    /// `timeout`. Repeated calls are safe and continue draining the same set.
2523    pub async fn drain_active_turns(&self, timeout: std::time::Duration) -> RuntimeDrainOutcome {
2524        let started_at = tokio::time::Instant::now();
2525        let persistence_failures_before =
2526            self.lifecycle_persistence_failures.load(Ordering::Acquire);
2527        let turn_admission = self.turn_admission.lock().await;
2528        self.accepting_turns.store(false, Ordering::Release);
2529        let active = self
2530            .active_turns
2531            .read()
2532            .await
2533            .iter()
2534            .map(|(turn_id, handle)| (turn_id.clone(), handle.thread_id.clone()))
2535            .collect::<Vec<_>>();
2536        let draining = self
2537            .turn_drains
2538            .read()
2539            .await
2540            .iter()
2541            .map(|(turn_id, drain)| (turn_id.clone(), drain.thread_id.clone()))
2542            .collect::<Vec<_>>();
2543        drop(turn_admission);
2544
2545        let mut interrupted_turns = std::collections::BTreeMap::new();
2546        for (turn_id, thread_id) in active.iter().chain(draining.iter()) {
2547            interrupted_turns.insert(turn_id.clone(), thread_id.clone());
2548        }
2549        let interrupted_turn_ids = interrupted_turns.keys().cloned().collect::<Vec<_>>();
2550        for (turn_id, thread_id) in active {
2551            let _ = self
2552                .interrupt_turn_with_reason(thread_id, turn_id, TurnLifecycleReason::Shutdown)
2553                .await;
2554        }
2555
2556        let wait_for_empty = async {
2557            loop {
2558                let notified = self.active_turns_changed.notified();
2559                if self.active_turns.read().await.is_empty()
2560                    && self.turn_drains.read().await.is_empty()
2561                {
2562                    return;
2563                }
2564                notified.await;
2565            }
2566        };
2567        let outcome = if tokio::time::timeout(timeout, wait_for_empty).await.is_ok() {
2568            if self.lifecycle_persistence_failures.load(Ordering::Acquire)
2569                > persistence_failures_before
2570            {
2571                RuntimeDrainOutcome::PersistenceFailed {
2572                    interrupted_turn_ids,
2573                    remaining_turn_ids: Vec::new(),
2574                }
2575            } else {
2576                RuntimeDrainOutcome::Clean {
2577                    interrupted_turn_ids,
2578                }
2579            }
2580        } else {
2581            let active_remaining = self
2582                .active_turns
2583                .read()
2584                .await
2585                .iter()
2586                .map(|(turn_id, handle)| (turn_id.clone(), handle.drain.clone()))
2587                .collect::<Vec<_>>();
2588            let draining_remaining = self
2589                .turn_drains
2590                .read()
2591                .await
2592                .iter()
2593                .map(|(turn_id, drain)| (turn_id.clone(), drain.clone()))
2594                .collect::<Vec<_>>();
2595            let remaining = active_remaining
2596                .into_iter()
2597                .chain(draining_remaining)
2598                .collect::<std::collections::BTreeMap<_, _>>();
2599            let remaining_turn_ids = remaining.keys().cloned().collect::<Vec<_>>();
2600            for turn_id in &remaining_turn_ids {
2601                let handle = remaining
2602                    .get(turn_id)
2603                    .expect("remaining turn ID must have a drain handle");
2604                self.record_turn_lifecycle(
2605                    handle.thread_id.clone(),
2606                    turn_id.clone(),
2607                    TurnLifecycleState::InterruptRequested,
2608                    TurnCleanupState::TimedOut,
2609                    Some(TurnLifecycleReason::Shutdown),
2610                )
2611                .await;
2612            }
2613            if self.lifecycle_persistence_failures.load(Ordering::Acquire)
2614                > persistence_failures_before
2615            {
2616                RuntimeDrainOutcome::PersistenceFailed {
2617                    interrupted_turn_ids,
2618                    remaining_turn_ids,
2619                }
2620            } else {
2621                RuntimeDrainOutcome::DeadlineExceeded {
2622                    interrupted_turn_ids,
2623                    remaining_turn_ids,
2624                }
2625            }
2626        };
2627        self.record_lifecycle_drain_outcome(started_at, &outcome);
2628        outcome
2629    }
2630
2631    /// Enables turn submission after a prior drain attempt. This is intended
2632    /// for embedders that keep a runtime alive after a bounded drain timeout.
2633    pub fn resume_accepting_turns(&self) {
2634        self.accepting_turns.store(true, Ordering::Release);
2635    }
2636
2637    pub async fn turn_lifecycle_snapshot(
2638        &self,
2639        thread_id: &ThreadId,
2640    ) -> anyhow::Result<TurnLifecycleSnapshot> {
2641        let Some(store) = &self.thread_store else {
2642            return Ok(TurnLifecycleSnapshot::default());
2643        };
2644        let extension_states = store.load_extension_states(thread_id).await?;
2645        Ok(turn_lifecycle_snapshot(&extension_states))
2646    }
2647
2648    pub async fn load_thread(
2649        &self,
2650        thread_id: &ThreadId,
2651    ) -> anyhow::Result<Option<ThreadSnapshot>> {
2652        let loaded = if let Some(store) = &self.thread_store {
2653            store.load_thread(thread_id).await?
2654        } else {
2655            None
2656        };
2657        if let Some(snapshot) = loaded.as_ref() {
2658            let live_turn_ids = self
2659                .active_turns
2660                .read()
2661                .await
2662                .keys()
2663                .cloned()
2664                .chain(self.turn_drains.read().await.keys().cloned())
2665                .collect::<std::collections::HashSet<_>>();
2666            let lifecycle = turn_lifecycle_snapshot(&snapshot.extension_states);
2667            for record in lifecycle.records.into_iter().filter(|record| {
2668                record.state.requires_recovery() && !live_turn_ids.contains(&record.turn_id)
2669            }) {
2670                self.lifecycle_restart_reconciliations
2671                    .fetch_add(1, Ordering::AcqRel);
2672                self.record_turn_lifecycle(
2673                    record.thread_id,
2674                    record.turn_id,
2675                    TurnLifecycleState::RecoveryNeeded,
2676                    TurnCleanupState::Unknown,
2677                    Some(TurnLifecycleReason::RuntimeRestart),
2678                )
2679                .await;
2680            }
2681            self.emit(RoderEvent::ThreadLoaded(ThreadLoaded {
2682                thread_id: thread_id.clone(),
2683                timestamp: OffsetDateTime::now_utc(),
2684            }))
2685            .await;
2686        }
2687        Ok(loaded)
2688    }
2689
2690    pub async fn workspace_for_thread(&self, thread_id: &ThreadId) -> anyhow::Result<String> {
2691        if let Some(store) = &self.thread_store {
2692            let snapshot = store
2693                .load_thread(thread_id)
2694                .await?
2695                .ok_or_else(|| anyhow::anyhow!("thread not found: {thread_id}"));
2696            match snapshot {
2697                Ok(snapshot) => {
2698                    if let Some(metadata) = snapshot.metadata {
2699                        // Fork-backed threads fail closed before any write
2700                        // when their workspace was removed out-of-band.
2701                        if let Some(fork) = &metadata.workspace_fork
2702                            && fork.status == roder_api::forks::ForkStatus::Active
2703                            && !std::path::Path::new(&metadata.workspace).is_dir()
2704                        {
2705                            anyhow::bail!(
2706                                "workspace fork {} is missing its workspace at {}; restore it or \
2707                                 remove the fork before running turns in this thread",
2708                                fork.id,
2709                                metadata.workspace
2710                            );
2711                        }
2712                        return Ok(metadata.workspace);
2713                    }
2714                    eprintln!(
2715                        "thread metadata missing while resolving workspace for {thread_id}; falling back to runtime workspace"
2716                    );
2717                }
2718                Err(err) => {
2719                    eprintln!(
2720                        "thread missing while resolving workspace for {thread_id}: {err}; falling back to runtime workspace"
2721                    );
2722                }
2723            }
2724        }
2725        Ok(self.workspace.display().to_string())
2726    }
2727
2728    pub(crate) async fn selection_mode_for_thread(
2729        &self,
2730        thread_id: &ThreadId,
2731    ) -> anyhow::Result<Option<ModelSelectionMode>> {
2732        let Some(store) = &self.thread_store else {
2733            return Ok(None);
2734        };
2735        Ok(store
2736            .load_thread(thread_id)
2737            .await?
2738            .and_then(|snapshot| snapshot.metadata)
2739            .and_then(|metadata| {
2740                metadata
2741                    .selection_mode
2742                    .or_else(|| match (metadata.provider, metadata.model) {
2743                        (Some(provider), Some(model)) => {
2744                            Some(ModelSelectionMode::manual(provider, model, None))
2745                        }
2746                        _ => None,
2747                    })
2748            }))
2749    }
2750
2751    /// Concrete provider/model for configured-role subagents (`task`,
2752    /// `agent_swarm`). Prefers the live parent turn selection so children stay
2753    /// on SuperGrok/Grok (or whatever the lead is using) instead of process
2754    /// startup defaults such as openai/gpt-5.5.
2755    pub(crate) async fn parent_model_selection_for_subagents(
2756        &self,
2757        thread_id: &ThreadId,
2758        turn_id: &TurnId,
2759    ) -> Option<roder_api::inference::ModelSelection> {
2760        if let Some(selection) = self
2761            .active_turn_selections
2762            .read()
2763            .await
2764            .get(turn_id)
2765            .cloned()
2766        {
2767            return Some(selection.concrete_selection());
2768        }
2769        if let Ok(Some(selection)) = self.selection_mode_for_thread(thread_id).await {
2770            return Some(selection.concrete_selection());
2771        }
2772        let cfg = self.status().await;
2773        if cfg.default_provider.trim().is_empty() || cfg.default_model.trim().is_empty() {
2774            return None;
2775        }
2776        Some(roder_api::inference::ModelSelection {
2777            provider: cfg.default_provider,
2778            model: cfg.default_model,
2779        })
2780    }
2781
2782    /// Per-thread tool allowlist, developer instructions, and external tools persisted at thread creation.
2783    pub(crate) async fn thread_turn_overrides(
2784        &self,
2785        thread_id: &ThreadId,
2786    ) -> anyhow::Result<ThreadTurnOverrides> {
2787        let Some(store) = &self.thread_store else {
2788            return Ok(ThreadTurnOverrides::default());
2789        };
2790        Ok(store
2791            .load_thread_metadata(thread_id)
2792            .await?
2793            .map(|metadata| ThreadTurnOverrides {
2794                tool_allowlist: metadata.tool_allowlist,
2795                developer_instructions: metadata.developer_instructions,
2796                external_tools: metadata.external_tools,
2797            })
2798            .unwrap_or_default())
2799    }
2800
2801    pub async fn set_thread_selection_mode(
2802        &self,
2803        thread_id: &ThreadId,
2804        selection_mode: ModelSelectionMode,
2805    ) -> anyhow::Result<()> {
2806        let Some(store) = &self.thread_store else {
2807            return Ok(());
2808        };
2809        let Some(snapshot) = store.load_thread(thread_id).await? else {
2810            anyhow::bail!("thread not found: {thread_id}");
2811        };
2812        let Some(mut metadata) = snapshot.metadata else {
2813            return Ok(());
2814        };
2815        let concrete = selection_mode.concrete_selection();
2816        metadata.provider = Some(concrete.provider);
2817        metadata.model = Some(concrete.model);
2818        metadata.selection_mode = Some(selection_mode);
2819        metadata.updated_at = OffsetDateTime::now_utc();
2820        store.update_thread_metadata(metadata).await?;
2821        Ok(())
2822    }
2823
2824    async fn runner_session_for_thread(
2825        &self,
2826        thread_id: &ThreadId,
2827    ) -> anyhow::Result<Option<(RunnerDestination, Arc<dyn RemoteRunnerSession>)>> {
2828        let metadata = if let Some(store) = &self.thread_store {
2829            store.load_thread_metadata(thread_id).await?
2830        } else {
2831            None
2832        };
2833        // An explicit per-thread binding wins over the runtime-level destination.
2834        let destination = metadata
2835            .as_ref()
2836            .and_then(|metadata| metadata.runner_binding.as_ref())
2837            .map(|binding| binding.destination.clone())
2838            .or(self.config.read().await.remote_runner_destination.clone());
2839        let Some(destination) = destination else {
2840            self.evict_runner_session(thread_id).await;
2841            return Ok(None);
2842        };
2843        let provider = self
2844            .registry
2845            .remote_runner_providers
2846            .iter()
2847            .find(|provider| provider.id() == destination.provider_id)
2848            .cloned()
2849            .ok_or_else(|| {
2850                anyhow::anyhow!(
2851                    "remote runner provider {:?} is not installed",
2852                    destination.provider_id
2853                )
2854            })?;
2855        let persisted_state = metadata.and_then(|metadata| metadata.runner_state);
2856        let slot = {
2857            let mut sessions = self.runner_sessions.lock().await;
2858            match sessions.get(thread_id) {
2859                Some(slot) if slot.matches(&destination) => slot.clone(),
2860                _ => {
2861                    let slot = Arc::new(RunnerSessionSlot::empty(&destination));
2862                    sessions.insert(thread_id.clone(), slot.clone());
2863                    slot
2864                }
2865            }
2866        };
2867
2868        loop {
2869            let initialized = slot.initialized.notified();
2870            let mut state = slot.state.lock().await;
2871            match &*state {
2872                RunnerSessionSlotState::Ready(cached) => {
2873                    return Ok(Some((cached.destination.clone(), cached.session.clone())));
2874                }
2875                RunnerSessionSlotState::Failed(error) => {
2876                    return Err(anyhow::anyhow!(error.to_string()));
2877                }
2878                RunnerSessionSlotState::Initializing => {
2879                    drop(state);
2880                    initialized.await;
2881                }
2882                RunnerSessionSlotState::Empty => {
2883                    *state = RunnerSessionSlotState::Initializing;
2884                    drop(state);
2885                    self.spawn_runner_session_initialization(
2886                        thread_id.clone(),
2887                        destination.clone(),
2888                        provider.clone(),
2889                        persisted_state.clone(),
2890                        slot.clone(),
2891                    );
2892                }
2893            }
2894        }
2895    }
2896
2897    fn spawn_runner_session_initialization(
2898        &self,
2899        thread_id: ThreadId,
2900        destination: RunnerDestination,
2901        provider: Arc<dyn RemoteRunnerProvider>,
2902        persisted_state: Option<RunnerSessionState>,
2903        slot: Arc<RunnerSessionSlot>,
2904    ) {
2905        let thread_store = self.thread_store.clone();
2906        let runner_sessions = self.runner_sessions.clone();
2907        // The task intentionally outlives the turn that first requested the
2908        // workspace. Interrupting that turn must not strand the slot in
2909        // `Initializing` and deadlock every later tool or lifecycle call.
2910        drop(tokio::spawn(async move {
2911            let initialized = async {
2912                let session = if let Some(state) = persisted_state
2913                    && state.provider_id == destination.provider_id
2914                    && state.destination_id == destination.id
2915                {
2916                    match provider.resume_session(state).await {
2917                        Ok(session) => session,
2918                        Err(_) => provider.create_session(destination.clone()).await?,
2919                    }
2920                } else {
2921                    provider.create_session(destination.clone()).await?
2922                };
2923                let resolved = (destination.clone(), session.clone());
2924                // Persist before releasing the singleflight or dispatching
2925                // the first tool. A later process can rejoin even if the turn
2926                // that requested initialization has already been interrupted.
2927                Self::persist_runner_state_in_store(
2928                    thread_store.as_ref(),
2929                    &thread_id,
2930                    Some(&resolved),
2931                )
2932                .await?;
2933                Ok::<_, anyhow::Error>(CachedRunnerSession {
2934                    destination,
2935                    session,
2936                })
2937            }
2938            .await;
2939
2940            match initialized {
2941                Ok(cached) => {
2942                    *slot.state.lock().await = RunnerSessionSlotState::Ready(cached);
2943                    slot.initialized.notify_waiters();
2944                }
2945                Err(error) => {
2946                    let message: Arc<str> = Arc::from(format!("{error:#}"));
2947                    *slot.state.lock().await = RunnerSessionSlotState::Failed(message);
2948                    slot.initialized.notify_waiters();
2949                    let mut sessions = runner_sessions.lock().await;
2950                    let is_current = sessions
2951                        .get(&thread_id)
2952                        .is_some_and(|current| Arc::ptr_eq(current, &slot));
2953                    if is_current {
2954                        sessions.remove(&thread_id);
2955                    }
2956                }
2957            }
2958        }));
2959    }
2960
2961    async fn evict_runner_session(&self, thread_id: &ThreadId) {
2962        self.runner_sessions.lock().await.remove(thread_id);
2963    }
2964
2965    async fn cache_runner_session(
2966        &self,
2967        thread_id: &ThreadId,
2968        destination: RunnerDestination,
2969        session: Arc<dyn RemoteRunnerSession>,
2970    ) {
2971        let cached = CachedRunnerSession {
2972            destination,
2973            session,
2974        };
2975        self.runner_sessions.lock().await.insert(
2976            thread_id.clone(),
2977            Arc::new(RunnerSessionSlot::ready(cached)),
2978        );
2979    }
2980
2981    /// Read a thread's explicit runner binding without creating or resuming a
2982    /// session. Tool routing uses this before local previews so runner-backed
2983    /// and fail-closed hosted calls never inspect the host workspace.
2984    pub(crate) async fn runner_binding_for_thread(
2985        &self,
2986        thread_id: &ThreadId,
2987    ) -> anyhow::Result<Option<ThreadRunnerBinding>> {
2988        let Some(store) = &self.thread_store else {
2989            return Ok(None);
2990        };
2991        Ok(store
2992            .load_thread_metadata(thread_id)
2993            .await?
2994            .and_then(|metadata| metadata.runner_binding))
2995    }
2996
2997    /// Resolve a previously-read binding into a live remote workspace. The
2998    /// session remains lazy: callers invoke this only after policy approval.
2999    pub(crate) async fn remote_workspace_for_binding(
3000        &self,
3001        thread_id: &ThreadId,
3002        binding: ThreadRunnerBinding,
3003    ) -> anyhow::Result<Arc<RemoteWorkspace>> {
3004        let session = self
3005            .runner_session_for_thread(thread_id)
3006            .await?
3007            .map(|(_, session)| session)
3008            .ok_or_else(|| {
3009                anyhow::anyhow!("runner-bound thread {thread_id} has no runner session")
3010            })?;
3011        Ok(Arc::new(RemoteWorkspace {
3012            session,
3013            root: binding.workspace,
3014            read_roots: binding.read_roots,
3015        }))
3016    }
3017
3018    pub(crate) async fn runner_tool_execution_lock(
3019        &self,
3020        session: &dyn RemoteRunnerSession,
3021    ) -> Arc<RunnerToolExecutionMutex> {
3022        let state = session.state();
3023        let key = (state.provider_id, state.session_id);
3024        let mut locks = self.runner_tool_execution_locks.lock().await;
3025        if let Some(lock) = locks.get(&key).and_then(Weak::upgrade) {
3026            return lock;
3027        }
3028
3029        locks.retain(|_, lock| lock.strong_count() > 0);
3030        let lock = Arc::new(Mutex::new(()));
3031        locks.insert(key, Arc::downgrade(&lock));
3032        lock
3033    }
3034
3035    async fn persist_runner_state(
3036        &self,
3037        thread_id: &ThreadId,
3038        runner: Option<&(RunnerDestination, Arc<dyn RemoteRunnerSession>)>,
3039    ) -> anyhow::Result<()> {
3040        Self::persist_runner_state_in_store(self.thread_store.as_ref(), thread_id, runner).await
3041    }
3042
3043    async fn persist_runner_state_in_store(
3044        store: Option<&Arc<dyn ThreadStore>>,
3045        thread_id: &ThreadId,
3046        runner: Option<&(RunnerDestination, Arc<dyn RemoteRunnerSession>)>,
3047    ) -> anyhow::Result<()> {
3048        let Some((destination, session)) = runner else {
3049            return Ok(());
3050        };
3051        let Some(store) = store else {
3052            return Ok(());
3053        };
3054        let Some(snapshot) = store.load_thread(thread_id).await? else {
3055            return Ok(());
3056        };
3057        let Some(mut metadata) = snapshot.metadata else {
3058            return Ok(());
3059        };
3060        metadata.runner_destination = Some(destination.clone());
3061        metadata.runner_state = Some(session.state());
3062        metadata.updated_at = OffsetDateTime::now_utc();
3063        store.update_thread_metadata(metadata).await?;
3064        Ok(())
3065    }
3066
3067    fn remote_runner_provider_by_id(
3068        &self,
3069        provider_id: &str,
3070    ) -> Option<Arc<dyn RemoteRunnerProvider>> {
3071        self.registry
3072            .remote_runner_providers
3073            .iter()
3074            .find(|provider| provider.id() == provider_id)
3075            .cloned()
3076    }
3077
3078    /// Resolve the live runner session for a thread along with its provider,
3079    /// failing clearly when the thread is not runner-bound.
3080    async fn thread_runner_session(
3081        &self,
3082        thread_id: &ThreadId,
3083    ) -> anyhow::Result<(
3084        RunnerDestination,
3085        Arc<dyn RemoteRunnerProvider>,
3086        Arc<dyn RemoteRunnerSession>,
3087    )> {
3088        let Some((destination, session)) = self.runner_session_for_thread(thread_id).await? else {
3089            anyhow::bail!("thread {thread_id} is not bound to a remote runner");
3090        };
3091        let provider = self
3092            .remote_runner_provider_by_id(&destination.provider_id)
3093            .ok_or_else(|| {
3094                anyhow::anyhow!(
3095                    "remote runner provider {:?} is not installed",
3096                    destination.provider_id
3097                )
3098            })?;
3099        Ok((destination, provider, session))
3100    }
3101
3102    /// Pause a runner-bound thread's session toward standby and persist the
3103    /// post-pause state. Errors if the provider is not pausable.
3104    pub async fn pause_thread_runner(
3105        &self,
3106        thread_id: &ThreadId,
3107    ) -> anyhow::Result<RunnerSessionState> {
3108        let (destination, provider, session) = self.thread_runner_session(thread_id).await?;
3109        anyhow::ensure!(
3110            provider.capabilities().pausable,
3111            "remote runner provider {:?} does not support pausing",
3112            destination.provider_id
3113        );
3114        let state = session.pause().await?;
3115        self.persist_runner_state(thread_id, Some(&(destination, session)))
3116            .await?;
3117        Ok(state)
3118    }
3119
3120    /// Resume (wake) a runner-bound thread's paused session and persist state.
3121    pub async fn resume_thread_runner(
3122        &self,
3123        thread_id: &ThreadId,
3124    ) -> anyhow::Result<RunnerSessionState> {
3125        let (destination, _provider, session) = self.thread_runner_session(thread_id).await?;
3126        let state = session.resume().await?;
3127        self.persist_runner_state(thread_id, Some(&(destination, session)))
3128            .await?;
3129        Ok(state)
3130    }
3131
3132    /// Detach a runner-bound thread's session: persist the durable, rejoinable
3133    /// state and leave the remote sandbox alive. Errors if not detachable.
3134    pub async fn detach_thread_runner(
3135        &self,
3136        thread_id: &ThreadId,
3137    ) -> anyhow::Result<RunnerSessionState> {
3138        let (destination, provider, session) = self.thread_runner_session(thread_id).await?;
3139        anyhow::ensure!(
3140            provider.capabilities().detachable,
3141            "remote runner provider {:?} does not support detaching",
3142            destination.provider_id
3143        );
3144        let state = session.detach().await?;
3145        // Persist the detached state explicitly so a later turn or process
3146        // rejoins the same sandbox instead of provisioning a new one.
3147        if let Some(store) = &self.thread_store
3148            && let Some(snapshot) = store.load_thread(thread_id).await?
3149            && let Some(mut metadata) = snapshot.metadata
3150        {
3151            metadata.runner_destination = Some(destination);
3152            metadata.runner_state = Some(state.clone());
3153            metadata.updated_at = OffsetDateTime::now_utc();
3154            store.update_thread_metadata(metadata).await?;
3155        }
3156        self.evict_runner_session(thread_id).await;
3157        Ok(state)
3158    }
3159
3160    /// Rejoin a previously created sandbox from a thread's persisted runner
3161    /// state without provisioning a new one. An optional `sandbox` overrides the
3162    /// persisted sandbox name (recovery by name). Persists refreshed state.
3163    pub async fn rejoin_thread_runner(
3164        &self,
3165        thread_id: &ThreadId,
3166        sandbox: Option<String>,
3167    ) -> anyhow::Result<RunnerSessionState> {
3168        let store = self
3169            .thread_store
3170            .as_ref()
3171            .ok_or_else(|| anyhow::anyhow!("thread store is required to rejoin a runner"))?;
3172        let metadata = store
3173            .load_thread_metadata(thread_id)
3174            .await?
3175            .ok_or_else(|| anyhow::anyhow!("thread {thread_id} has no metadata"))?;
3176        let destination = metadata
3177            .runner_binding
3178            .as_ref()
3179            .map(|binding| binding.destination.clone())
3180            .or_else(|| metadata.runner_destination.clone())
3181            .ok_or_else(|| anyhow::anyhow!("thread {thread_id} is not bound to a remote runner"))?;
3182        let mut state = metadata
3183            .runner_state
3184            .clone()
3185            .ok_or_else(|| anyhow::anyhow!("thread {thread_id} has no persisted runner state"))?;
3186        if let Some(sandbox) = sandbox
3187            && let Some(object) = state.metadata.as_object_mut()
3188        {
3189            object.insert("sandbox_name".to_string(), serde_json::Value::from(sandbox));
3190        }
3191        let provider = self
3192            .remote_runner_provider_by_id(&destination.provider_id)
3193            .ok_or_else(|| {
3194                anyhow::anyhow!(
3195                    "remote runner provider {:?} is not installed",
3196                    destination.provider_id
3197                )
3198            })?;
3199        let session = provider.rejoin_session(state).await?;
3200        self.persist_runner_state(thread_id, Some(&(destination.clone(), session.clone())))
3201            .await?;
3202        self.cache_runner_session(thread_id, destination, session.clone())
3203            .await;
3204        Ok(session.state())
3205    }
3206
3207    pub(crate) async fn record_thread_usage_metadata(
3208        &self,
3209        thread_id: &ThreadId,
3210        usage: &TokenUsage,
3211    ) -> anyhow::Result<()> {
3212        if usage.is_empty() {
3213            return Ok(());
3214        }
3215        let Some(store) = &self.thread_store else {
3216            return Ok(());
3217        };
3218        let Some(snapshot) = store.load_thread(thread_id).await? else {
3219            return Ok(());
3220        };
3221        let Some(mut metadata) = snapshot.metadata else {
3222            return Ok(());
3223        };
3224        metadata
3225            .usage
3226            .get_or_insert_with(ThreadUsageMetadata::default)
3227            .add_token_usage(usage);
3228        metadata.updated_at = OffsetDateTime::now_utc();
3229        store.update_thread_metadata(metadata).await?;
3230        Ok(())
3231    }
3232
3233    pub fn start_turn(
3234        self: &Arc<Self>,
3235        mut req: StartTurnRequest,
3236    ) -> BoxFuture<'_, anyhow::Result<TurnId>> {
3237        Box::pin(async move {
3238            let _thread_admission = self.thread_admission(&req.thread_id).await;
3239            let turn_admission = self.turn_admission.lock().await;
3240            anyhow::ensure!(
3241                self.accepting_turns.load(Ordering::Acquire),
3242                "runtime is quiescing and cannot accept new turns"
3243            );
3244            req.workspace = validate_thread_workspace(&req.workspace)?;
3245            let team_member = self.teams.member_for_thread(&req.thread_id).await;
3246            let cfg = self.config.read().await.clone();
3247            let provider = req
3248                .provider_override
3249                .clone()
3250                .unwrap_or_else(|| cfg.default_provider.clone());
3251            self.engine_for(&provider)?;
3252            let turn_id = uuid::Uuid::new_v4().to_string();
3253            let mut initial_mailbox_ack = None;
3254            if let Some((team_id, member)) = &team_member {
3255                let pending = self
3256                    .teams
3257                    .reserve_pending_mailbox_messages(team_id, &member.id, &turn_id)
3258                    .await?;
3259                if !pending.is_empty()
3260                    && let Some(team) = self.read_team(team_id).await
3261                {
3262                    let mailbox = format_mailbox_messages(&team, &pending);
3263                    req.message = if req.message.trim().is_empty() {
3264                        mailbox
3265                    } else {
3266                        format!("{mailbox}\n\n[Direct task input]\n{}", req.message)
3267                    };
3268                    initial_mailbox_ack = Some(MailboxDeliveryAck {
3269                        team_id: team_id.clone(),
3270                        message_ids: pending.iter().map(|message| message.id.clone()).collect(),
3271                    });
3272                }
3273            }
3274            let (abort_handle, abort_registration) = AbortHandle::new_pair();
3275            let drain = Arc::new(TurnDrainHandle {
3276                thread_id: req.thread_id.clone(),
3277                interrupt_requested: AtomicBool::new(false),
3278                interrupt_reason: Mutex::new(None),
3279                completed: AtomicBool::new(false),
3280                completed_notify: Notify::new(),
3281            });
3282            let (steer_changed, steering) = tokio::sync::watch::channel(0);
3283            let active = ActiveTurnHandle {
3284                thread_id: req.thread_id.clone(),
3285                abort: abort_handle,
3286                steers: Arc::new(Mutex::new(Vec::new())),
3287                steer_changed,
3288                drain,
3289            };
3290            self.active_turns
3291                .write()
3292                .await
3293                .insert(turn_id.clone(), active);
3294            self.record_turn_lifecycle(
3295                req.thread_id.clone(),
3296                turn_id.clone(),
3297                TurnLifecycleState::Running,
3298                TurnCleanupState::NotRequested,
3299                None,
3300            )
3301            .await;
3302            self.active_turn_contexts.write().await.insert(
3303                turn_id.clone(),
3304                InheritedTurnContext {
3305                    workspace: req.workspace.clone(),
3306                    instructions: req.instructions.clone(),
3307                    developer_context: req.developer_context.clone(),
3308                },
3309            );
3310            if let Some((team_id, member)) = team_member {
3311                let updated = match self
3312                    .teams
3313                    .update_member(&team_id, &member.id, |member| {
3314                        member.current_turn_id = Some(turn_id.clone());
3315                        member.status = TeamMemberStatus::Running;
3316                        member.final_message = None;
3317                        member.terminal_error = None;
3318                    })
3319                    .await
3320                {
3321                    Ok(updated) => updated,
3322                    Err(error) => {
3323                        self.active_turns.write().await.remove(&turn_id);
3324                        self.active_turn_contexts.write().await.remove(&turn_id);
3325                        self.teams
3326                            .release_mailbox_reservations_for_turn(&turn_id)
3327                            .await;
3328                        return Err(error);
3329                    }
3330                };
3331                if let Some(member) = updated
3332                    .members
3333                    .into_iter()
3334                    .find(|candidate| candidate.id == member.id)
3335                {
3336                    self.emit(RoderEvent::TeamMemberStatusChanged(
3337                        TeamMemberStatusChanged {
3338                            team_id,
3339                            member_id: member.id,
3340                            member_thread_id: member.thread_id,
3341                            status: TeamMemberStatus::Running,
3342                            timestamp: OffsetDateTime::now_utc(),
3343                        },
3344                    ))
3345                    .await;
3346                }
3347            }
3348            let runtime = Arc::clone(self);
3349            let turn_req = req;
3350            let thread_id_for_task = turn_req.thread_id.clone();
3351            let turn_id_for_task = turn_id.clone();
3352            tokio::spawn(async move {
3353                let result = Abortable::new(
3354                    runtime.run_turn(
3355                        turn_req,
3356                        turn_id_for_task.clone(),
3357                        initial_mailbox_ack,
3358                        steering,
3359                    ),
3360                    abort_registration,
3361                )
3362                .await;
3363                /*
3364                 * A failed sibling in a parallel tool batch drops in-flight external tool
3365                 * futures (`try_join_all` in `route_tool_calls`), stranding their
3366                 * `pending_external_tool_calls` entries. Sweep before reporting the turn
3367                 * outcome so every `thread/toolExecutionRequested` gets a terminal
3368                 * resolution; on clean completion the map holds nothing for this turn.
3369                 */
3370                runtime
3371                    .cancel_pending_external_tool_calls_for_turn(&turn_id_for_task)
3372                    .await;
3373                let completed = matches!(&result, Ok(Ok(TurnRunOutcome::Completed)));
3374                match &result {
3375                    Ok(Err(err)) => {
3376                        let (cleanup, ownership) =
3377                            runtime.await_provider_turn_cleanup(&turn_id_for_task).await;
3378                        // This wrapper owns terminal failure emission for returned errors.
3379                        runtime
3380                            .emit(RoderEvent::TurnFailed(TurnFailed {
3381                                thread_id: thread_id_for_task.clone(),
3382                                turn_id: turn_id_for_task.clone(),
3383                                error: err.to_string(),
3384                                error_kind: err
3385                                    .downcast_ref::<roder_api::provider_error::ProviderFailure>()
3386                                    .map(|failure| failure.kind.retry_cause().to_string()),
3387                                usage: None,
3388                                timestamp: OffsetDateTime::now_utc(),
3389                            }))
3390                            .await;
3391                        runtime
3392                            .record_turn_lifecycle_with_ownership(
3393                                thread_id_for_task.clone(),
3394                                turn_id_for_task.clone(),
3395                                TurnLifecycleState::Failed,
3396                                cleanup,
3397                                Some(TurnLifecycleReason::ProviderFailure),
3398                                ownership,
3399                            )
3400                            .await;
3401                        let _ = runtime
3402                            .complete_team_member_turn_with_result(
3403                                &thread_id_for_task,
3404                                &turn_id_for_task,
3405                                TeamMemberStatus::Failed,
3406                                None,
3407                                Some(err.to_string()),
3408                            )
3409                            .await;
3410                    }
3411                    Ok(Ok(TurnRunOutcome::Stopped)) => {
3412                        let (cleanup, ownership) =
3413                            runtime.await_provider_turn_cleanup(&turn_id_for_task).await;
3414                        runtime
3415                            .record_turn_lifecycle_with_ownership(
3416                                thread_id_for_task.clone(),
3417                                turn_id_for_task.clone(),
3418                                TurnLifecycleState::Failed,
3419                                cleanup,
3420                                Some(TurnLifecycleReason::ProviderFailure),
3421                                ownership,
3422                            )
3423                            .await;
3424                        let _ = runtime
3425                            .complete_team_member_turn_with_result(
3426                                &thread_id_for_task,
3427                                &turn_id_for_task,
3428                                TeamMemberStatus::Failed,
3429                                None,
3430                                Some("turn stopped before completion".to_string()),
3431                            )
3432                            .await;
3433                    }
3434                    Err(_) => {
3435                        let reason = if let Some(handle) = runtime
3436                            .turn_drains
3437                            .read()
3438                            .await
3439                            .get(&turn_id_for_task)
3440                            .cloned()
3441                        {
3442                            handle
3443                                .interrupt_reason
3444                                .lock()
3445                                .await
3446                                .unwrap_or(TurnLifecycleReason::RuntimeFailure)
3447                        } else {
3448                            TurnLifecycleReason::RuntimeFailure
3449                        };
3450                        let (cleanup, ownership) =
3451                            runtime.await_provider_turn_cleanup(&turn_id_for_task).await;
3452                        runtime
3453                            .record_turn_lifecycle_with_ownership(
3454                                thread_id_for_task.clone(),
3455                                turn_id_for_task.clone(),
3456                                TurnLifecycleState::Interrupted,
3457                                cleanup,
3458                                Some(reason),
3459                                ownership,
3460                            )
3461                            .await;
3462                        runtime
3463                            .emit(RoderEvent::TurnInterrupted(TurnInterrupted {
3464                                thread_id: thread_id_for_task.clone(),
3465                                turn_id: turn_id_for_task.clone(),
3466                                timestamp: OffsetDateTime::now_utc(),
3467                            }))
3468                            .await;
3469                        let _ = runtime
3470                            .complete_team_member_turn_with_result(
3471                                &thread_id_for_task,
3472                                &turn_id_for_task,
3473                                TeamMemberStatus::Interrupted,
3474                                None,
3475                                None,
3476                            )
3477                            .await;
3478                    }
3479                    Ok(Ok(TurnRunOutcome::Completed)) => {}
3480                }
3481                runtime
3482                    .teams
3483                    .release_mailbox_reservations_for_turn(&turn_id_for_task)
3484                    .await;
3485                runtime.active_turns.write().await.remove(&turn_id_for_task);
3486                if let Some(drain) = runtime.turn_drains.write().await.remove(&turn_id_for_task) {
3487                    drain.completed.store(true, Ordering::Release);
3488                    drain.completed_notify.notify_waiters();
3489                }
3490                runtime
3491                    .active_turn_selections
3492                    .write()
3493                    .await
3494                    .remove(&turn_id_for_task);
3495                runtime
3496                    .active_turn_contexts
3497                    .write()
3498                    .await
3499                    .remove(&turn_id_for_task);
3500                if !completed {
3501                    // Completion paths above consume registered provider cleanup
3502                    // handles. A setup failure before an engine can stream has no
3503                    // such handle; this is a harmless final defensive sweep.
3504                    let _ = runtime.provider_turn_cleanups.lock().map(|mut cleanups| {
3505                        cleanups.remove(&turn_id_for_task);
3506                    });
3507                }
3508                runtime.active_turns_changed.notify_waiters();
3509                if completed {
3510                    let _ = runtime
3511                        .continue_active_goal_after_turn(thread_id_for_task)
3512                        .await;
3513                }
3514            });
3515            drop(turn_admission);
3516            Ok(turn_id)
3517        })
3518    }
3519
3520    pub(crate) async fn has_active_turn_for_thread(&self, thread_id: &ThreadId) -> bool {
3521        self.active_turns
3522            .read()
3523            .await
3524            .values()
3525            .any(|handle| &handle.thread_id == thread_id)
3526    }
3527
3528    async fn ensure_codex_v2_team_capacity(
3529        &self,
3530        team: &TeamState,
3531        resuming_member_id: &str,
3532        candidate_model: Option<&str>,
3533    ) -> anyhow::Result<()> {
3534        if !is_codex_v2_team(team)
3535            && !candidate_model
3536                .is_some_and(|model| model_supports_reasoning_effort(model, REASONING_ULTRA))
3537        {
3538            return Ok(());
3539        }
3540        let active_thread_ids = self
3541            .active_turns
3542            .read()
3543            .await
3544            .values()
3545            .map(|handle| handle.thread_id.clone())
3546            .collect::<std::collections::HashSet<_>>();
3547        let resident_threads = 1 + team
3548            .members
3549            .iter()
3550            .filter(|member| {
3551                member.role != roder_api::teams::TeamMemberRole::Lead
3552                    && member.id != resuming_member_id
3553                    && active_thread_ids.contains(&member.thread_id)
3554            })
3555            .count();
3556        anyhow::ensure!(
3557            resident_threads < codex_v2::CODEX_V2_MAX_RESIDENT_TEAM_THREADS,
3558            "agent thread limit reached for this Codex V2 team: maximum {} resident threads (the lead plus 3 running subagents)",
3559            codex_v2::CODEX_V2_MAX_RESIDENT_TEAM_THREADS
3560        );
3561        Ok(())
3562    }
3563
3564    /// Number of currently running turns across all threads. Hosted runtime
3565    /// pools use this to avoid evicting tenants with active work.
3566    pub async fn active_turn_count(&self) -> usize {
3567        self.active_turns.read().await.len()
3568    }
3569
3570    pub async fn active_turn_for_thread(&self, thread_id: &ThreadId) -> Option<TurnId> {
3571        self.active_turns
3572            .read()
3573            .await
3574            .iter()
3575            .find_map(|(turn_id, handle)| (&handle.thread_id == thread_id).then(|| turn_id.clone()))
3576    }
3577
3578    pub async fn thread_activity(&self, thread_id: &ThreadId) -> ThreadActivity {
3579        let active_turn_id = self.active_turn_for_thread(thread_id).await;
3580        let draining_turn_id = if active_turn_id.is_none() {
3581            self.turn_drains
3582                .read()
3583                .await
3584                .iter()
3585                .find_map(|(turn_id, drain)| {
3586                    (&drain.thread_id == thread_id).then(|| turn_id.clone())
3587                })
3588        } else {
3589            None
3590        };
3591        let Some(active_turn_id) = active_turn_id.or(draining_turn_id) else {
3592            return ThreadActivity::default();
3593        };
3594
3595        let mut active_flags = Vec::new();
3596        {
3597            let pending_approvals = self.pending_tool_approvals.lock().await;
3598            if pending_approvals
3599                .values()
3600                .any(|pending| &pending.thread_id == thread_id && pending.turn_id == active_turn_id)
3601            {
3602                active_flags.push("approvalRequired".to_string());
3603            }
3604        }
3605        {
3606            let pending_inputs = self.pending_user_inputs.lock().await;
3607            if pending_inputs
3608                .values()
3609                .any(|pending| &pending.thread_id == thread_id && pending.turn_id == active_turn_id)
3610            {
3611                active_flags.push("userInputRequired".to_string());
3612            }
3613        }
3614        {
3615            let pending_external = self.pending_external_tool_calls.lock().await;
3616            if pending_external
3617                .values()
3618                .any(|pending| &pending.thread_id == thread_id && pending.turn_id == active_turn_id)
3619            {
3620                active_flags.push("externalToolPending".to_string());
3621            }
3622        }
3623        let interrupting = self
3624            .active_turns
3625            .read()
3626            .await
3627            .get(&active_turn_id)
3628            .is_some_and(|handle| handle.drain.interrupt_requested.load(Ordering::Acquire))
3629            || self
3630                .turn_drains
3631                .read()
3632                .await
3633                .get(&active_turn_id)
3634                .is_some_and(|drain| drain.interrupt_requested.load(Ordering::Acquire));
3635        if interrupting {
3636            active_flags.push("interrupting".to_string());
3637        }
3638        if self.pending_plan_exit().await.is_some_and(|pending| {
3639            &pending.thread_id == thread_id && pending.turn_id == active_turn_id
3640        }) {
3641            active_flags.push("planExitRequired".to_string());
3642        }
3643
3644        ThreadActivity {
3645            active_turn_id: Some(active_turn_id),
3646            active_flags,
3647        }
3648    }
3649
3650    pub async fn interrupt_turn(&self, thread_id: ThreadId, turn_id: TurnId) -> anyhow::Result<()> {
3651        self.interrupt_turn_with_reason(thread_id, turn_id, TurnLifecycleReason::UserInterrupt)
3652            .await
3653    }
3654
3655    async fn interrupt_turn_with_reason(
3656        &self,
3657        thread_id: ThreadId,
3658        turn_id: TurnId,
3659        reason: TurnLifecycleReason,
3660    ) -> anyhow::Result<()> {
3661        let handle = self.active_turns.write().await.remove(&turn_id);
3662        if let Some(handle) = handle {
3663            if handle
3664                .drain
3665                .interrupt_requested
3666                .swap(true, Ordering::AcqRel)
3667            {
3668                return Ok(());
3669            }
3670            *handle.drain.interrupt_reason.lock().await = Some(reason);
3671            self.turn_drains
3672                .write()
3673                .await
3674                .insert(turn_id.clone(), handle.drain.clone());
3675            let ownership = self.provider_cleanup_ownership(&turn_id);
3676            self.record_turn_lifecycle_with_ownership(
3677                thread_id.clone(),
3678                turn_id.clone(),
3679                TurnLifecycleState::InterruptRequested,
3680                TurnCleanupState::Requested,
3681                Some(reason),
3682                ownership,
3683            )
3684            .await;
3685            handle.abort.abort();
3686            self.active_turns_changed.notify_waiters();
3687        }
3688        self.active_turn_selections.write().await.remove(&turn_id);
3689        self.cancel_pending_external_tool_calls_for_turn(&turn_id)
3690            .await;
3691        Ok(())
3692    }
3693
3694    pub async fn steer_turn(
3695        &self,
3696        thread_id: ThreadId,
3697        turn_id: TurnId,
3698        message: String,
3699        images: Vec<InputImage>,
3700    ) -> anyhow::Result<()> {
3701        self.enqueue_turn_steer(thread_id, turn_id, message, images, None)
3702            .await
3703    }
3704
3705    pub(crate) async fn has_pending_turn_steers(&self, turn_id: &TurnId) -> bool {
3706        let active = self.active_turns.read().await.get(turn_id).cloned();
3707        let Some(active) = active else {
3708            return false;
3709        };
3710        let has_pending = !active.steers.lock().await.is_empty();
3711        has_pending
3712    }
3713
3714    async fn steer_turn_with_mailbox_ack(
3715        &self,
3716        thread_id: ThreadId,
3717        turn_id: TurnId,
3718        message: String,
3719        message_ids: Vec<String>,
3720        team_id: TeamId,
3721    ) -> anyhow::Result<()> {
3722        self.enqueue_turn_steer(
3723            thread_id,
3724            turn_id,
3725            message,
3726            Vec::new(),
3727            Some(MailboxDeliveryAck {
3728                team_id,
3729                message_ids,
3730            }),
3731        )
3732        .await
3733    }
3734
3735    async fn deliver_pending_team_mailbox(
3736        &self,
3737        team_id: &str,
3738        member_id: &str,
3739        member_thread_id: ThreadId,
3740        turn_id: TurnId,
3741    ) -> anyhow::Result<()> {
3742        let pending = self
3743            .teams
3744            .reserve_pending_mailbox_messages(team_id, member_id, &turn_id)
3745            .await?;
3746        if pending.is_empty() {
3747            return Ok(());
3748        }
3749        let message_ids = pending
3750            .iter()
3751            .map(|message| message.id.clone())
3752            .collect::<Vec<_>>();
3753        let team = self
3754            .read_team(team_id)
3755            .await
3756            .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
3757        if let Err(error) = self
3758            .steer_turn_with_mailbox_ack(
3759                member_thread_id,
3760                turn_id.clone(),
3761                format_mailbox_messages(&team, &pending),
3762                message_ids.clone(),
3763                team_id.to_string(),
3764            )
3765            .await
3766        {
3767            self.teams
3768                .release_mailbox_reservations(&turn_id, &message_ids)
3769                .await;
3770            return Err(error);
3771        }
3772        Ok(())
3773    }
3774
3775    async fn enqueue_turn_steer(
3776        &self,
3777        thread_id: ThreadId,
3778        turn_id: TurnId,
3779        message: String,
3780        images: Vec<InputImage>,
3781        mailbox_ack: Option<MailboxDeliveryAck>,
3782    ) -> anyhow::Result<()> {
3783        let message = message.trim().to_string();
3784        if message.is_empty() && images.is_empty() {
3785            return Ok(());
3786        }
3787
3788        let Some(active) = self.active_turns.read().await.get(&turn_id).cloned() else {
3789            anyhow::bail!("no active turn to steer");
3790        };
3791        anyhow::ensure!(
3792            active.thread_id == thread_id,
3793            "turn does not belong to thread"
3794        );
3795        {
3796            let mut steers = active.steers.lock().await;
3797            steers.push(QueuedTurnSteer {
3798                message: UserMessage::with_images(message.clone(), images),
3799                mailbox_ack,
3800            });
3801            active
3802                .steer_changed
3803                .send_modify(|generation| *generation = generation.wrapping_add(1));
3804        }
3805        self.emit(RoderEvent::TurnSteered(TurnSteered {
3806            thread_id,
3807            turn_id,
3808            message,
3809            timestamp: OffsetDateTime::now_utc(),
3810        }))
3811        .await;
3812        Ok(())
3813    }
3814
3815    pub async fn tool_specs(&self) -> Vec<roder_api::tools::ToolSpec> {
3816        let cfg = self.config.read().await;
3817        let model_profile =
3818            model_profile_for_provider_model(&cfg, &cfg.default_provider, &cfg.default_model);
3819        self.filtered_tool_specs(
3820            &cfg,
3821            &cfg.default_model,
3822            &cfg.default_provider,
3823            model_profile.as_ref(),
3824            &[],
3825            &[],
3826        )
3827    }
3828
3829    pub fn subagent_definitions(&self) -> Vec<SubagentDefinition> {
3830        self.registry
3831            .subagent_dispatchers
3832            .iter()
3833            .flat_map(|dispatcher| dispatcher.definitions())
3834            .collect()
3835    }
3836
3837    async fn run_turn(
3838        self: &Arc<Self>,
3839        req: StartTurnRequest,
3840        turn_id: TurnId,
3841        initial_mailbox_ack: Option<MailboxDeliveryAck>,
3842        mut steering: tokio::sync::watch::Receiver<u64>,
3843    ) -> anyhow::Result<TurnRunOutcome> {
3844        let turn_started_at = OffsetDateTime::now_utc();
3845        self.emit(RoderEvent::TurnStarted(TurnStarted {
3846            thread_id: req.thread_id.clone(),
3847            turn_id: turn_id.clone(),
3848            runtime_profile: self.config.read().await.runtime_profile,
3849            timestamp: turn_started_at,
3850        }))
3851        .await;
3852        self.persist_turn_item(
3853            &req.thread_id,
3854            &turn_id,
3855            &TranscriptItem::UserMessage(UserMessage::with_images(
3856                req.message.clone(),
3857                req.images.clone(),
3858            )),
3859        )
3860        .await?;
3861        crate::hooks::run_lifecycle(
3862            self,
3863            &req.thread_id,
3864            &turn_id,
3865            Some(&req.workspace),
3866            "UserPromptSubmit",
3867            None,
3868            serde_json::json!({"prompt": req.message}),
3869        )
3870        .await;
3871        if let Some(ack) = initial_mailbox_ack {
3872            self.teams
3873                .mark_mailbox_messages_delivered(&ack.team_id, &turn_id, &ack.message_ids)
3874                .await?;
3875        }
3876
3877        let mut cfg = self.config.read().await.clone();
3878        let runtime_profile = cfg.runtime_profile;
3879        let turn_deadline = turn_deadline_for_config(&cfg);
3880        let deadline_finalization_reserve =
3881            crate::deadline_policy::finalization_reserve_seconds(cfg.turn_deadline_seconds);
3882        let selection_mode = self.selection_mode_for_thread(&req.thread_id).await?;
3883        let concrete_selection = selection_mode
3884            .as_ref()
3885            .map(ModelSelectionMode::concrete_selection);
3886        let default_provider = req
3887            .provider_override
3888            .clone()
3889            .or_else(|| {
3890                concrete_selection
3891                    .as_ref()
3892                    .map(|selection| selection.provider.clone())
3893            })
3894            .unwrap_or(cfg.default_provider.clone());
3895        let default_model = req
3896            .model_override
3897            .clone()
3898            .or_else(|| {
3899                concrete_selection
3900                    .as_ref()
3901                    .map(|selection| selection.model.clone())
3902            })
3903            .unwrap_or(cfg.default_model.clone());
3904        if let Some(reasoning) = req.reasoning_override.as_deref().or_else(|| {
3905            selection_mode
3906                .as_ref()
3907                .and_then(ModelSelectionMode::reasoning)
3908        }) {
3909            validate_reasoning_effort(&default_model, reasoning)?;
3910            cfg.reasoning = Some(reasoning.to_string());
3911        }
3912        let turn_has_concrete_model_override =
3913            req.provider_override.is_some() || req.model_override.is_some();
3914        let (turn_inference_router, turn_inference_router_profile) = match &selection_mode {
3915            Some(ModelSelectionMode::Auto {
3916                router_id, profile, ..
3917            }) if !turn_has_concrete_model_override => (
3918                RuntimeInferenceRouterConfig {
3919                    enabled: true,
3920                    router_id: Some(router_id.clone()),
3921                },
3922                profile.clone(),
3923            ),
3924            _ => (RuntimeInferenceRouterConfig::disabled(), None),
3925        };
3926        let mut provider = default_provider.clone();
3927        let mut model = default_model.clone();
3928        let mut model_profile = model_profile_for_provider_model(&cfg, &provider, &model);
3929        let workspace = req.workspace.clone();
3930        let compaction_generation_at_start = self.compaction_generation(&req.thread_id);
3931        let mut transcript = self.transcript_for_turn(&req, &turn_id, &model).await?;
3932        let mut compacted_this_turn =
3933            self.compaction_generation(&req.thread_id) != compaction_generation_at_start;
3934        let effective_policy_mode = self.effective_policy_mode_for_thread(&req.thread_id).await;
3935        let agent_swarm_mode_active = self
3936            .effective_agent_swarm_mode_for_thread(&req.thread_id)
3937            .await;
3938        let ultra_mode_active = self.effective_ultra_mode_for_thread(&req.thread_id).await;
3939        let thread_overrides = self.thread_turn_overrides(&req.thread_id).await?;
3940        let mut final_assistant_text = String::new();
3941        let mut final_provider_metadata = None;
3942        let mut exhausted_tool_rounds = true;
3943        let mut verification_gate =
3944            VerificationGateState::new(req.message.clone(), runtime_profile);
3945        let mut speed_policy = SpeedPolicyState::default();
3946        let mut reliability = TurnReliabilityState::default();
3947        let mut turn_usage = TokenUsage::default();
3948        // Overwritten on every inference step's Completed event so only the
3949        // terminal step's stop reason survives (mid-turn tool_use steps must
3950        // not leak onto turn/completed).
3951        let mut turn_finish_reason: Option<String> = None;
3952        let mut deadline_finalization_requested = false;
3953        let mut deadline_scoreable_completion_requested = false;
3954        let mut task_ledger_completion_reminders = 0_u8;
3955        let mut task_ledger_scoreable_checkpoints = 0_u8;
3956        let mut empty_tool_call_nudges_used = 0_u32;
3957        let mut provider_stream_retry_attempts = 0_u32;
3958        let mut context_limit_recovery_attempts = 0_u32;
3959        let mut routing_candidates = None;
3960        let routing_transcript_start = transcript.len().saturating_sub(1);
3961        let mut routing_escalations = 0_u32;
3962        let mut model_switch_summary_selection = None::<ModelSelection>;
3963
3964        'tool_rounds: for round_index in 0..MAX_TOOL_ROUNDS_PER_TURN {
3965            if let Some(deadline) = turn_deadline
3966                && deadline_expired(deadline)
3967            {
3968                self.fail_turn_due_to_deadline(&req.thread_id, &turn_id, deadline, &transcript)
3969                    .await?;
3970                return Ok(TurnRunOutcome::Stopped);
3971            }
3972            let steers = self.drain_turn_steers(&turn_id, &mut steering).await;
3973            self.append_steers(&req, &turn_id, &mut transcript, steers)
3974                .await?;
3975            if runtime_profile == RuntimeProfile::Eval
3976                && let Some(remaining) = crate::deadline_policy::should_start_finalization(
3977                    turn_deadline,
3978                    deadline_finalization_reserve,
3979                    deadline_finalization_requested || deadline_scoreable_completion_requested,
3980                )
3981            {
3982                if req.task_ledger_required
3983                    && task_ledger_completion_reminders < TASK_LEDGER_COMPLETION_REMINDER_LIMIT
3984                    && let Some(prompt) = task_ledger_completion_prompt(&transcript)
3985                {
3986                    task_ledger_completion_reminders += 1;
3987                    deadline_scoreable_completion_requested = true;
3988                    let item = TranscriptItem::UserMessage(UserMessage::text(
3989                        task_ledger_deadline_completion_prompt(
3990                            remaining,
3991                            deadline_finalization_reserve,
3992                            &prompt,
3993                        ),
3994                    ));
3995                    self.persist_turn_item(&req.thread_id, &turn_id, &item)
3996                        .await?;
3997                    transcript.push(item);
3998                    continue 'tool_rounds;
3999                } else {
4000                    self.start_deadline_finalization(
4001                        &req.thread_id,
4002                        &turn_id,
4003                        &mut transcript,
4004                        remaining,
4005                    )
4006                    .await?;
4007                    deadline_finalization_requested = true;
4008                }
4009            }
4010            if runtime_profile == RuntimeProfile::Eval
4011                && req.task_ledger_required
4012                && !deadline_finalization_requested
4013                && task_ledger_scoreable_checkpoints < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
4014                && let Some(remaining) = deadline_remaining_seconds(turn_deadline)
4015                && remaining <= TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS
4016                && remaining > deadline_finalization_reserve
4017                && let Some(prompt) = task_ledger_completion_prompt(&transcript)
4018            {
4019                task_ledger_scoreable_checkpoints += 1;
4020                let item = TranscriptItem::UserMessage(UserMessage::text(
4021                    task_ledger_scoreable_checkpoint_prompt(remaining, &prompt),
4022                ));
4023                self.persist_turn_item(&req.thread_id, &turn_id, &item)
4024                    .await?;
4025                transcript.push(item);
4026                continue 'tool_rounds;
4027            }
4028            if turn_inference_router.is_active() && routing_candidates.is_none() {
4029                routing_candidates =
4030                    Some(collect_inference_routing_candidates(&self.registry).await);
4031            }
4032            let routing_tools_model = model.clone();
4033            let routing_tools_provider = provider.clone();
4034            let routing_tools = self.filtered_tool_specs(
4035                &cfg,
4036                &model,
4037                &provider,
4038                model_profile.as_ref(),
4039                &thread_overrides.tool_allowlist,
4040                &thread_overrides.external_tools,
4041            );
4042            let prior_failures =
4043                transcript_failure_count_since(&transcript, routing_transcript_start)
4044                    .max(reliability.tool_failure_count())
4045                    .saturating_add(provider_stream_retry_attempts);
4046            let routing_selection = route_inference_selection(
4047                &self.registry,
4048                &turn_inference_router,
4049                InferenceRoutingRequest {
4050                    thread_id: &req.thread_id,
4051                    turn_id: &turn_id,
4052                    round_index: round_index as u32,
4053                    runtime_profile,
4054                    phase: speed_policy.phase(),
4055                    profile: turn_inference_router_profile.as_deref(),
4056                    default_selection: ModelSelection {
4057                        provider: default_provider.clone(),
4058                        model: default_model.clone(),
4059                    },
4060                    transcript: &transcript,
4061                    tools: &routing_tools,
4062                    candidates: routing_candidates.as_deref(),
4063                    prior_failures,
4064                    prior_escalations: routing_escalations,
4065                },
4066            )
4067            .await;
4068            if let Some(decision) = routing_selection.decision.clone() {
4069                if matches!(decision.outcome, InferenceRoutingOutcome::Escalated) {
4070                    routing_escalations = routing_escalations.saturating_add(1);
4071                }
4072                self.emit(RoderEvent::InferenceRoutingDecision(
4073                    InferenceRoutingDecisionEvent {
4074                        thread_id: req.thread_id.clone(),
4075                        turn_id: turn_id.clone(),
4076                        round_index: round_index as u32,
4077                        default_selection: ModelSelection {
4078                            provider: default_provider.clone(),
4079                            model: default_model.clone(),
4080                        },
4081                        selected_selection: routing_selection.selection.clone(),
4082                        decision,
4083                        timestamp: OffsetDateTime::now_utc(),
4084                    },
4085                ))
4086                .await;
4087            }
4088            provider = routing_selection.selection.provider.clone();
4089            model = routing_selection.selection.model.clone();
4090            let engine = self.engine_for(&provider)?;
4091            let capabilities = engine.capabilities();
4092            model_profile = model_profile_for_provider_model(&cfg, &provider, &model);
4093            let tools = if capabilities.tool_calls {
4094                if model == routing_tools_model && provider == routing_tools_provider {
4095                    routing_tools.clone()
4096                } else {
4097                    self.filtered_tool_specs(
4098                        &cfg,
4099                        &model,
4100                        &provider,
4101                        model_profile.as_ref(),
4102                        &thread_overrides.tool_allowlist,
4103                        &thread_overrides.external_tools,
4104                    )
4105                }
4106            } else {
4107                Vec::new()
4108            };
4109            let parallel_tool_calls = parallel_tool_calls_for_model(&cfg, &model);
4110            let tool_choice = if tools.is_empty() {
4111                ToolChoice::None
4112            } else {
4113                ToolChoice::Auto
4114            };
4115            let summary_selection = ModelSelection {
4116                provider: provider.clone(),
4117                model: model.clone(),
4118            };
4119            if model_switch_summary_selection.as_ref() != Some(&summary_selection) {
4120                if let Some(summary) = model_switch_summary(
4121                    &transcript,
4122                    model_profile.as_ref(),
4123                    &provider,
4124                    &model,
4125                    &tools,
4126                ) {
4127                    let item = TranscriptItem::UserMessage(UserMessage::text(summary));
4128                    self.persist_turn_item(&req.thread_id, &turn_id, &item)
4129                        .await?;
4130                    transcript.push(item);
4131                }
4132                model_switch_summary_selection = Some(summary_selection);
4133            }
4134
4135            if !capabilities.image_input && transcript_has_images(&transcript) {
4136                self.fail_turn_with_error(
4137                    &req.thread_id,
4138                    &turn_id,
4139                    format!("provider {provider} does not support image input"),
4140                )
4141                .await?;
4142                return Ok(TurnRunOutcome::Stopped);
4143            }
4144            let compaction_generation_before = self.compaction_generation(&req.thread_id);
4145            transcript = self
4146                .compact_transcript_if_needed(
4147                    &req.thread_id,
4148                    &turn_id,
4149                    &provider,
4150                    &model,
4151                    transcript,
4152                    self.compaction_options_for_turn(&req.thread_id, !compacted_this_turn),
4153                )
4154                .await?;
4155            compacted_this_turn |=
4156                self.compaction_generation(&req.thread_id) != compaction_generation_before;
4157
4158            let speed_policy_decision =
4159                speed_policy.decision(runtime_profile, &model, &cfg.speed_policy);
4160            let request_reasoning = reasoning_from_decision(
4161                speed_policy_decision.as_ref(),
4162                routing_selection
4163                    .reasoning
4164                    .clone()
4165                    .unwrap_or_else(|| reasoning_for_model(&cfg, &model)),
4166            );
4167            self.active_turn_selections.write().await.insert(
4168                turn_id.clone(),
4169                ModelSelectionMode::manual(
4170                    provider.clone(),
4171                    model.clone(),
4172                    request_reasoning.level.clone(),
4173                ),
4174            );
4175            if let Some(limit) = reliability.record_model_call(
4176                &cfg.reliability,
4177                runtime_profile == RuntimeProfile::Interactive,
4178            ) {
4179                self.fail_turn_due_to_reliability_limit(
4180                    &req.thread_id,
4181                    &turn_id,
4182                    &provider,
4183                    &model,
4184                    limit,
4185                    &transcript,
4186                )
4187                .await?;
4188                return Ok(TurnRunOutcome::Stopped);
4189            }
4190            self.emit(RoderEvent::InferenceStarted(InferenceStarted {
4191                thread_id: req.thread_id.clone(),
4192                turn_id: turn_id.clone(),
4193                engine_id: engine.id(),
4194                model: ModelSelection {
4195                    provider: provider.clone(),
4196                    model: model.clone(),
4197                },
4198                reasoning: request_reasoning.clone(),
4199                speed_policy: speed_policy_decision.clone(),
4200                deadline_remaining_seconds: deadline_remaining_seconds(turn_deadline),
4201                timestamp: OffsetDateTime::now_utc(),
4202            }))
4203            .await;
4204
4205            let mut instructions = req.instructions.clone();
4206            if let Some(extra) = &thread_overrides.developer_instructions {
4207                instructions = apply_thread_developer_instructions(instructions, extra);
4208            }
4209            if let Some(context) = req.developer_context.as_deref() {
4210                instructions = apply_turn_developer_context(instructions, context);
4211            }
4212            let mut instructions = apply_runtime_profile(instructions, runtime_profile);
4213            if let Some(profile) = &model_profile {
4214                instructions = apply_model_instruction_overlay(instructions, profile);
4215            }
4216            if req.task_ledger_required
4217                && runtime_profile == RuntimeProfile::Eval
4218                && !transcript_has_task_ledger(&transcript)
4219            {
4220                instructions = apply_task_ledger_required(instructions);
4221            }
4222            if effective_policy_mode == PolicyMode::Plan {
4223                instructions = apply_plan_mode(instructions);
4224            }
4225            if agent_swarm_mode_active {
4226                instructions = apply_agent_swarm_mode(instructions);
4227            }
4228            // Ultra mode (any model) enables proactive multi-agent policy.
4229            // Codex Sol/Terra Ultra effort still does too without requiring the
4230            // mode flag. Models that advertise Ultra effort keep the
4231            // explicit-request-only policy on lower efforts when ultra mode is
4232            // off; other models get multi-agent policy only when ultra mode is on.
4233            let model_has_ultra_effort = model_supports_reasoning_effort(&model, REASONING_ULTRA);
4234            let effort_is_ultra = request_reasoning.level.as_deref() == Some(REASONING_ULTRA);
4235            let proactive_multi_agent = ultra_mode_active || effort_is_ultra;
4236            if ultra_mode_active || model_has_ultra_effort {
4237                instructions = apply_codex_multi_agent_mode(instructions, proactive_multi_agent);
4238            }
4239            instructions = self
4240                .goals
4241                .apply_goal_instructions(&req.thread_id, instructions)
4242                .await?;
4243            instructions =
4244                apply_parallel_web_tools(instructions, tools.iter().map(|spec| spec.name.as_str()));
4245            let mut request_metadata = serde_json::json!({});
4246            if let Some(decision) = &speed_policy_decision {
4247                request_metadata["speedPolicy"] = serde_json::json!(decision);
4248            }
4249            if let Some(decision) = routing_selection.decision.as_ref() {
4250                request_metadata["inferenceRouting"] = serde_json::json!(decision);
4251            }
4252            if let Some(remaining) = deadline_remaining_seconds(turn_deadline) {
4253                request_metadata["deadlineRemainingSeconds"] = serde_json::json!(remaining);
4254            }
4255            if let Some(profile) = &model_profile {
4256                request_metadata["modelProfile"] = serde_json::json!({
4257                    "model": profile.model,
4258                    "providerFamily": profile.provider_family,
4259                    "editTool": profile.edit_tool,
4260                    "schemaPolicy": profile.schema_policy,
4261                    "instructionOverlay": profile.instruction_overlay,
4262                    "parallelToolCalls": profile.parallel_tool_calls,
4263                    "autoCompactTokenLimit": profile.auto_compact_token_limit,
4264                });
4265            }
4266            let task_ledger_required_this_round = req.task_ledger_required
4267                && runtime_profile == RuntimeProfile::Eval
4268                && !deadline_finalization_requested
4269                && !transcript_has_task_ledger(&transcript);
4270            let task_ledger_tools = (capabilities.tool_calls && task_ledger_required_this_round)
4271                .then(|| self.task_ledger_tool_specs(model_profile.as_ref()))
4272                .filter(|tools| !tools.is_empty());
4273            let request_tools = if deadline_finalization_requested {
4274                Vec::new()
4275            } else if let Some(ledger_tools) = &task_ledger_tools {
4276                ledger_tools.clone()
4277            } else {
4278                tools.clone()
4279            };
4280            let request_tool_choice = if deadline_finalization_requested {
4281                ToolChoice::None
4282            } else if task_ledger_tools.is_some() {
4283                ToolChoice::Specific(TASK_LEDGER_TOOL_NAME.to_string())
4284            } else {
4285                tool_choice.clone()
4286            };
4287            if deadline_finalization_requested {
4288                request_metadata["deadlineFinalization"] = serde_json::json!({
4289                    "reserveSeconds": deadline_finalization_reserve,
4290                    "remainingSeconds": deadline_remaining_seconds(turn_deadline),
4291                });
4292            } else if deadline_scoreable_completion_requested {
4293                request_metadata["deadlineScoreableCompletion"] = serde_json::json!({
4294                    "reserveSeconds": deadline_finalization_reserve,
4295                    "remainingSeconds": deadline_remaining_seconds(turn_deadline),
4296                });
4297            }
4298            let mut eager_tools = EagerTools::new(
4299                self,
4300                &request_tools,
4301                &thread_overrides.external_tools,
4302                parallel_tool_calls,
4303            );
4304            let mut request_reliability: ReliabilityRequestPolicy = cfg.reliability.clone().into();
4305            request_reliability.provider_retry_max_attempts = request_reliability
4306                .provider_retry_max_attempts
4307                .saturating_sub(provider_stream_retry_attempts)
4308                .max(1);
4309            let request = AgentInferenceRequest {
4310                model: ModelSelection {
4311                    provider: provider.clone(),
4312                    model: model.clone(),
4313                },
4314                instructions,
4315                transcript: transcript.clone(),
4316                tools: request_tools,
4317                tool_choice: request_tool_choice,
4318                reasoning: request_reasoning,
4319                output: OutputConfig::default(),
4320                runtime: RuntimeHints {
4321                    auto_compact_token_limit: (provider != roder_api::catalog::PROVIDER_CODEX)
4322                        .then(|| server_side_compaction_threshold(&cfg, &model))
4323                        .flatten(),
4324                    profile: runtime_profile,
4325                    parallel_tool_calls: Some(parallel_tool_calls),
4326                    prompt_cache_key: Some(req.thread_id.clone()),
4327                    hosted_web_search: cfg.hosted_web_search.clone(),
4328                    tool_search: tool_search_for_provider_model(&cfg, &provider, &model),
4329                    speed_policy: speed_policy_decision,
4330                    reliability: Some(request_reliability),
4331                    deadline_remaining_seconds: deadline_remaining_seconds(turn_deadline),
4332                    service_tier: req.service_tier_override.clone(),
4333                    ..RuntimeHints::default()
4334                },
4335                metadata: request_metadata,
4336            };
4337
4338            let mut compaction_template = request.clone();
4339            compaction_template.transcript.clear();
4340            self.compaction_templates
4341                .write()
4342                .await
4343                .insert(req.thread_id.clone(), compaction_template);
4344
4345            let ctx = InferenceTurnContext {
4346                thread_id: &req.thread_id,
4347                turn_id: &turn_id,
4348                tool_executor: Some(std::sync::Arc::new(
4349                    crate::tool_execution::RuntimeTurnToolExecutor {
4350                        runtime: Arc::clone(self),
4351                        thread_id: req.thread_id.clone(),
4352                        turn_id: turn_id.clone(),
4353                        workspace: Some(workspace.clone()),
4354                        deadline: turn_deadline,
4355                    },
4356                )),
4357            };
4358            let stream_future = preemptible(engine.stream_turn(ctx, request), &mut steering);
4359            let mut stream = if let Some((deadline, timeout_action)) = inference_timeout_deadline(
4360                turn_deadline,
4361                runtime_profile,
4362                req.task_ledger_required,
4363                deadline_finalization_reserve,
4364                deadline_finalization_requested || deadline_scoreable_completion_requested,
4365                task_ledger_scoreable_checkpoints,
4366                &transcript,
4367            ) {
4368                match tokio::time::timeout_at(deadline_instant(deadline), stream_future).await {
4369                    Ok(stream) => match stream {
4370                        Ok(stream) => stream,
4371                        Err(err) => {
4372                            if err.is::<SamplingPreempted>() {
4373                                self.finish_sampling_cleanup(&turn_id).await?;
4374                                provider_stream_retry_attempts = 0;
4375                                continue 'tool_rounds;
4376                            }
4377                            self.finish_sampling_cleanup(&turn_id).await?;
4378                            if !deadline_finalization_requested
4379                                && self
4380                                    .retry_sampling_failure(
4381                                        SamplingRetry {
4382                                            thread_id: &req.thread_id,
4383                                            turn_id: &turn_id,
4384                                            provider: &provider,
4385                                            model: &model,
4386                                            config: &cfg.reliability,
4387                                            error: &err,
4388                                        },
4389                                        &mut provider_stream_retry_attempts,
4390                                        &mut steering,
4391                                    )
4392                                    .await
4393                            {
4394                                continue 'tool_rounds;
4395                            }
4396                            let error = err.to_string();
4397                            if self
4398                                .try_recover_from_context_limit(
4399                                    &req.thread_id,
4400                                    &turn_id,
4401                                    &provider,
4402                                    &model,
4403                                    &error,
4404                                    &mut transcript,
4405                                    &mut compacted_this_turn,
4406                                    &mut context_limit_recovery_attempts,
4407                                )
4408                                .await?
4409                            {
4410                                continue 'tool_rounds;
4411                            }
4412                            return Err(err);
4413                        }
4414                    },
4415                    Err(_) => {
4416                        if runtime_profile == RuntimeProfile::Eval
4417                            && !deadline_finalization_requested
4418                        {
4419                            let remaining = deadline_remaining_seconds(turn_deadline).unwrap_or(0);
4420                            if timeout_action == InferenceTimeoutAction::ScoreableCheckpoint
4421                                && task_ledger_scoreable_checkpoints
4422                                    < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
4423                                && let Some(prompt) = task_ledger_completion_prompt(&transcript)
4424                            {
4425                                task_ledger_scoreable_checkpoints += 1;
4426                                let item = TranscriptItem::UserMessage(UserMessage::text(
4427                                    task_ledger_scoreable_checkpoint_prompt(remaining, &prompt),
4428                                ));
4429                                self.persist_turn_item(&req.thread_id, &turn_id, &item)
4430                                    .await?;
4431                                transcript.push(item);
4432                                continue 'tool_rounds;
4433                            }
4434                            self.start_deadline_finalization(
4435                                &req.thread_id,
4436                                &turn_id,
4437                                &mut transcript,
4438                                remaining,
4439                            )
4440                            .await?;
4441                            deadline_finalization_requested = true;
4442                            continue 'tool_rounds;
4443                        }
4444                        self.fail_turn_due_to_deadline(
4445                            &req.thread_id,
4446                            &turn_id,
4447                            deadline,
4448                            &transcript,
4449                        )
4450                        .await?;
4451                        return Ok(TurnRunOutcome::Stopped);
4452                    }
4453                }
4454            } else {
4455                match stream_future.await {
4456                    Ok(stream) => stream,
4457                    Err(err) => {
4458                        if err.is::<SamplingPreempted>() {
4459                            self.finish_sampling_cleanup(&turn_id).await?;
4460                            provider_stream_retry_attempts = 0;
4461                            continue 'tool_rounds;
4462                        }
4463                        self.finish_sampling_cleanup(&turn_id).await?;
4464                        if !deadline_finalization_requested
4465                            && self
4466                                .retry_sampling_failure(
4467                                    SamplingRetry {
4468                                        thread_id: &req.thread_id,
4469                                        turn_id: &turn_id,
4470                                        provider: &provider,
4471                                        model: &model,
4472                                        config: &cfg.reliability,
4473                                        error: &err,
4474                                    },
4475                                    &mut provider_stream_retry_attempts,
4476                                    &mut steering,
4477                                )
4478                                .await
4479                        {
4480                            continue 'tool_rounds;
4481                        }
4482                        let error = err.to_string();
4483                        if self
4484                            .try_recover_from_context_limit(
4485                                &req.thread_id,
4486                                &turn_id,
4487                                &provider,
4488                                &model,
4489                                &error,
4490                                &mut transcript,
4491                                &mut compacted_this_turn,
4492                                &mut context_limit_recovery_attempts,
4493                            )
4494                            .await?
4495                        {
4496                            continue 'tool_rounds;
4497                        }
4498                        return Err(err);
4499                    }
4500                }
4501            };
4502            let mut sampling_output = SamplingOutput::default();
4503            let output_context = OutputContext {
4504                thread_id: &req.thread_id,
4505                turn_id: &turn_id,
4506                profile: model_profile.as_ref(),
4507                provider: &provider,
4508                model: &model,
4509            };
4510            let mut assistant_text = String::new();
4511            let mut phase_messages = Vec::<AssistantMessage>::new();
4512            let mut reasoning_text = String::new();
4513            let mut replayed_tool_call = false;
4514            let mut tool_calls = Vec::new();
4515            let mut patch_progress = PatchProgressTracker::default();
4516            let mut provider_metadata = None;
4517            let mut provider_requests_continuation = false;
4518            // Captured from mid-stream Compaction(completed) events. Codex/OpenAI
4519            // sometimes abort the SSE stream after emitting the compaction item
4520            // ("error decoding response body") before response.completed, so we
4521            // must not rely solely on ProviderMetadata for the boundary.
4522            let mut stream_compaction_item: Option<serde_json::Value> = None;
4523
4524            loop {
4525                let next_future = async {
4526                    loop {
4527                        tokio::select! {
4528                            biased;
4529                            _ = wait_for_steer(&mut steering) => break Some(Err(SamplingPreempted.into())),
4530                            result = eager_tools.poll_result(), if eager_tools.pending() => {
4531                                if let Err(error) = result { break Some(Err(error)); }
4532                            }
4533                            next = stream.next() => break next,
4534                        }
4535                    }
4536                };
4537                let next = if let Some((deadline, timeout_action)) = inference_timeout_deadline(
4538                    turn_deadline,
4539                    runtime_profile,
4540                    req.task_ledger_required,
4541                    deadline_finalization_reserve,
4542                    deadline_finalization_requested || deadline_scoreable_completion_requested,
4543                    task_ledger_scoreable_checkpoints,
4544                    &transcript,
4545                ) {
4546                    match tokio::time::timeout_at(deadline_instant(deadline), next_future).await {
4547                        Ok(next) => next,
4548                        Err(_) => {
4549                            if runtime_profile == RuntimeProfile::Eval
4550                                && !deadline_finalization_requested
4551                            {
4552                                let remaining =
4553                                    deadline_remaining_seconds(turn_deadline).unwrap_or(0);
4554                                if timeout_action == InferenceTimeoutAction::ScoreableCheckpoint
4555                                    && task_ledger_scoreable_checkpoints
4556                                        < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
4557                                    && let Some(prompt) = task_ledger_completion_prompt(&transcript)
4558                                {
4559                                    task_ledger_scoreable_checkpoints += 1;
4560                                    let item = TranscriptItem::UserMessage(UserMessage::text(
4561                                        task_ledger_scoreable_checkpoint_prompt(remaining, &prompt),
4562                                    ));
4563                                    self.persist_turn_item(&req.thread_id, &turn_id, &item)
4564                                        .await?;
4565                                    transcript.push(item);
4566                                    continue 'tool_rounds;
4567                                }
4568                                self.start_deadline_finalization(
4569                                    &req.thread_id,
4570                                    &turn_id,
4571                                    &mut transcript,
4572                                    remaining,
4573                                )
4574                                .await?;
4575                                deadline_finalization_requested = true;
4576                                continue 'tool_rounds;
4577                            }
4578                            self.fail_turn_due_to_deadline(
4579                                &req.thread_id,
4580                                &turn_id,
4581                                deadline,
4582                                &transcript,
4583                            )
4584                            .await?;
4585                            return Ok(TurnRunOutcome::Stopped);
4586                        }
4587                    }
4588                } else {
4589                    next_future.await
4590                };
4591                let res = next.unwrap_or_else(|| {
4592                    Err(roder_api::provider_error::ProviderFailure::new(
4593                        roder_api::provider_error::ProviderFailureKind::StreamInterrupted,
4594                        "provider stream ended before terminal completion",
4595                    )
4596                    .into())
4597                });
4598                let event = match res {
4599                    Ok(event) => event,
4600                    Err(err) => {
4601                        drop(stream);
4602                        eager_tools.drain().await?;
4603                        for result in eager_tools.take_results(&tool_calls) {
4604                            verification_gate.record_tool_result(&result);
4605                            transcript.push(TranscriptItem::ToolResult(result));
4606                        }
4607                        self.finish_sampling_cleanup(&turn_id).await?;
4608                        if err.is::<SamplingPreempted>() {
4609                            provider_stream_retry_attempts = 0;
4610                            continue 'tool_rounds;
4611                        }
4612                        if !deadline_finalization_requested
4613                            && self
4614                                .retry_sampling_failure(
4615                                    SamplingRetry {
4616                                        thread_id: &req.thread_id,
4617                                        turn_id: &turn_id,
4618                                        provider: &provider,
4619                                        model: &model,
4620                                        config: &cfg.reliability,
4621                                        error: &err,
4622                                    },
4623                                    &mut provider_stream_retry_attempts,
4624                                    &mut steering,
4625                                )
4626                                .await
4627                        {
4628                            continue 'tool_rounds;
4629                        }
4630                        let error = err.to_string();
4631                        if self
4632                            .try_recover_from_context_limit(
4633                                &req.thread_id,
4634                                &turn_id,
4635                                &provider,
4636                                &model,
4637                                &error,
4638                                &mut transcript,
4639                                &mut compacted_this_turn,
4640                                &mut context_limit_recovery_attempts,
4641                            )
4642                            .await?
4643                        {
4644                            continue 'tool_rounds;
4645                        }
4646                        return Err(err);
4647                    }
4648                };
4649
4650                let inference_timestamp = OffsetDateTime::now_utc();
4651                self.emit(RoderEvent::InferenceEventReceived(InferenceEventReceived {
4652                    thread_id: req.thread_id.clone(),
4653                    turn_id: turn_id.clone(),
4654                    event: event.clone(),
4655                    timestamp: inference_timestamp,
4656                }))
4657                .await;
4658
4659                match event {
4660                    InferenceEvent::MessageDelta(delta) => {
4661                        if let Some((team_id, member)) =
4662                            self.teams.member_for_thread(&req.thread_id).await
4663                        {
4664                            self.emit(RoderEvent::TeamMemberMessageDelta(TeamMemberMessageDelta {
4665                                team_id,
4666                                member_id: member.id,
4667                                member_thread_id: req.thread_id.clone(),
4668                                turn_id: turn_id.clone(),
4669                                delta: delta.text.clone(),
4670                                timestamp: OffsetDateTime::now_utc(),
4671                            }))
4672                            .await;
4673                        }
4674                        if is_final_answer_phase(delta.phase.as_deref()) {
4675                            assistant_text.push_str(&delta.text);
4676                        } else if let Some(last) = phase_messages.last_mut()
4677                            && last.phase == delta.phase
4678                        {
4679                            last.text.push_str(&delta.text);
4680                        } else {
4681                            phase_messages.push(AssistantMessage {
4682                                text: delta.text,
4683                                phase: delta.phase,
4684                            });
4685                        }
4686                    }
4687                    InferenceEvent::ReasoningDelta(delta) => reasoning_text.push_str(&delta.text),
4688                    InferenceEvent::ToolCallCompleted(call) => {
4689                        if completed_call_replayed(&transcript, &call)? {
4690                            replayed_tool_call = true;
4691                            continue;
4692                        }
4693                        if let Some(progress) = patch_progress.complete(
4694                            &req.thread_id,
4695                            &turn_id,
4696                            &call.id,
4697                            &call.name,
4698                            &call.arguments,
4699                        ) {
4700                            self.emit(RoderEvent::PatchProgress(progress)).await;
4701                        }
4702                        if !tool_calls
4703                            .iter()
4704                            .any(|previous: &ToolCallCompleted| previous.id == call.id)
4705                        {
4706                            eager_tools
4707                                .schedule(
4708                                    self,
4709                                    &output_context,
4710                                    &call,
4711                                    workspace.as_str(),
4712                                    turn_deadline,
4713                                    &mut transcript,
4714                                )
4715                                .await?;
4716                            tool_calls.push(call);
4717                        }
4718                    }
4719                    InferenceEvent::OutputItemCompleted(item) => {
4720                        sampling_output
4721                            .complete_item(self, &output_context, item, &mut transcript)
4722                            .await?;
4723                    }
4724                    InferenceEvent::Failed(failure) => {
4725                        speed_policy.record_failure();
4726                        let error = failure.message;
4727                        if self
4728                            .try_recover_from_context_limit(
4729                                &req.thread_id,
4730                                &turn_id,
4731                                &provider,
4732                                &model,
4733                                &error,
4734                                &mut transcript,
4735                                &mut compacted_this_turn,
4736                                &mut context_limit_recovery_attempts,
4737                            )
4738                            .await?
4739                        {
4740                            continue 'tool_rounds;
4741                        }
4742                        self.persist_turn_item(
4743                            &req.thread_id,
4744                            &turn_id,
4745                            &TranscriptItem::Error(ErrorRecord {
4746                                message: error.clone(),
4747                            }),
4748                        )
4749                        .await?;
4750                        self.emit(RoderEvent::TurnFailed(TurnFailed {
4751                            thread_id: req.thread_id.clone(),
4752                            turn_id: turn_id.clone(),
4753                            error: error.clone(),
4754                            error_kind: None,
4755                            usage: None,
4756                            timestamp: OffsetDateTime::now_utc(),
4757                        }))
4758                        .await;
4759                        self.complete_team_member_turn_with_result(
4760                            &req.thread_id,
4761                            &turn_id,
4762                            TeamMemberStatus::Failed,
4763                            (!assistant_text.trim().is_empty()).then(|| assistant_text.clone()),
4764                            Some(error),
4765                        )
4766                        .await?;
4767                        return Ok(TurnRunOutcome::Stopped);
4768                    }
4769                    InferenceEvent::Usage(usage) => {
4770                        turn_usage.add_assign(&usage);
4771                    }
4772                    InferenceEvent::Completed(metadata) => {
4773                        sampling_output
4774                            .finish(
4775                                self,
4776                                &output_context,
4777                                &assistant_text,
4778                                &phase_messages,
4779                                &reasoning_text,
4780                                &mut transcript,
4781                            )
4782                            .await?;
4783                        provider_stream_retry_attempts = 0;
4784                        turn_finish_reason = metadata
4785                            .stop_reason
4786                            .as_deref()
4787                            .map(finish_reason_from_stop_reason);
4788                        break;
4789                    }
4790                    InferenceEvent::Compaction(progress) => {
4791                        // Persist the boundary as soon as the provider emits a
4792                        // completed compaction item. ChatGPT Codex often aborts
4793                        // the SSE stream right after ("error decoding response
4794                        // body"), so waiting for ProviderMetadata loses the
4795                        // boundary and every subsequent request re-compacts.
4796                        if progress.status == "completed"
4797                            && let Some(item) = progress.item
4798                        {
4799                            let already = stream_compaction_item
4800                                .as_ref()
4801                                .and_then(|existing| {
4802                                    existing.get("id").and_then(serde_json::Value::as_str)
4803                                })
4804                                .zip(item.get("id").and_then(serde_json::Value::as_str))
4805                                .is_some_and(|(a, b)| a == b);
4806                            if !already {
4807                                stream_compaction_item = Some(item.clone());
4808                                let boundary = TranscriptItem::ProviderMetadata(
4809                                    crate::compaction::provider_metadata_from_compaction_item(item),
4810                                );
4811                                self.persist_turn_item(&req.thread_id, &turn_id, &boundary)
4812                                    .await?;
4813                                transcript.push(boundary);
4814                                transcript =
4815                                    crate::compaction::trim_to_last_compaction_boundary(transcript);
4816                                compacted_this_turn = true;
4817                            }
4818                        }
4819                    }
4820                    InferenceEvent::HostedToolCallStarted(_)
4821                    | InferenceEvent::HostedToolCallCompleted(_) => {}
4822                    InferenceEvent::ToolCallStarted(call) => {
4823                        patch_progress.start(&call.id, &call.name)
4824                    }
4825                    InferenceEvent::ToolCallDelta(delta) => {
4826                        if let Some(progress) = patch_progress.delta(
4827                            &req.thread_id,
4828                            &turn_id,
4829                            &delta.id,
4830                            &delta.arguments_delta,
4831                        ) {
4832                            self.emit(RoderEvent::PatchProgress(progress)).await;
4833                        }
4834                    }
4835                    InferenceEvent::ProviderMetadata(mut metadata) => {
4836                        if metadata.get("kind").and_then(serde_json::Value::as_str)
4837                            == Some("reliability_retry_attempt")
4838                        {
4839                            provider_stream_retry_attempts =
4840                                provider_stream_retry_attempts.saturating_add(1);
4841                        }
4842                        provider_requests_continuation |= metadata
4843                            .get("end_turn")
4844                            .and_then(serde_json::Value::as_bool)
4845                            == Some(false);
4846                        if let Some(compaction_item) = stream_compaction_item.as_ref() {
4847                            let _ = crate::compaction::ensure_provider_metadata_compaction(
4848                                &mut metadata,
4849                                compaction_item,
4850                            );
4851                        }
4852                        if metadata.get("output").is_none() {
4853                            let item = TranscriptItem::ProviderMetadata(metadata);
4854                            self.persist_turn_item(&req.thread_id, &turn_id, &item)
4855                                .await?;
4856                            transcript.push(item);
4857                        } else {
4858                            provider_metadata = Some(metadata);
4859                        }
4860                    }
4861                }
4862            }
4863
4864            speed_policy.record_model_output(
4865                !assistant_text.is_empty() || !phase_messages.is_empty(),
4866                tool_calls.len(),
4867            );
4868            if tool_calls.is_empty() {
4869                if provider_requests_continuation || replayed_tool_call {
4870                    if let Some(metadata) = provider_metadata {
4871                        let item = TranscriptItem::ProviderMetadata(metadata);
4872                        self.persist_turn_item(&req.thread_id, &turn_id, &item)
4873                            .await?;
4874                        transcript.push(item);
4875                    }
4876                    continue 'tool_rounds;
4877                }
4878                let steers = self.drain_turn_steers(&turn_id, &mut steering).await;
4879                if !steers.is_empty() {
4880                    if let Some(metadata) = provider_metadata {
4881                        let had_provider_compaction =
4882                            crate::compaction::provider_metadata_has_compaction(&metadata);
4883                        let item = TranscriptItem::ProviderMetadata(metadata);
4884                        self.persist_turn_item(&req.thread_id, &turn_id, &item)
4885                            .await?;
4886                        transcript.push(item);
4887                        if had_provider_compaction {
4888                            // Server-side compaction replaced the prior window;
4889                            // drop pre-boundary items so the next request does
4890                            // not re-send (and re-compact) the full history.
4891                            transcript =
4892                                crate::compaction::trim_to_last_compaction_boundary(transcript);
4893                            compacted_this_turn = true;
4894                        }
4895                    }
4896                    self.append_steers(&req, &turn_id, &mut transcript, steers)
4897                        .await?;
4898                    continue;
4899                }
4900                if !deadline_finalization_requested
4901                    && req.task_ledger_required
4902                    && runtime_profile == RuntimeProfile::Eval
4903                    && task_ledger_completion_reminders < TASK_LEDGER_COMPLETION_REMINDER_LIMIT
4904                    && (!assistant_text.trim().is_empty() || !phase_messages.is_empty())
4905                    && let Some(prompt) = task_ledger_completion_prompt(&transcript)
4906                {
4907                    task_ledger_completion_reminders += 1;
4908                    let item = TranscriptItem::UserMessage(UserMessage::text(prompt));
4909                    self.persist_turn_item(&req.thread_id, &turn_id, &item)
4910                        .await?;
4911                    transcript.push(item);
4912                    continue;
4913                }
4914                if !deadline_finalization_requested
4915                    && let Some(prompt) = verification_gate.blocking_prompt()
4916                {
4917                    speed_policy.record_verification_required();
4918                    self.emit(RoderEvent::VerificationRequired(VerificationRequired {
4919                        thread_id: req.thread_id.clone(),
4920                        turn_id: turn_id.clone(),
4921                        reason: verification_gate.reason(),
4922                        changed_files: verification_gate.changed_files(),
4923                        tool_evidence: verification_gate.tool_evidence.clone(),
4924                        tests_run: verification_gate.tests_run.clone(),
4925                        open_gaps: verification_gate.open_gaps.clone(),
4926                        timestamp: OffsetDateTime::now_utc(),
4927                    }))
4928                    .await;
4929                    let item = TranscriptItem::UserMessage(UserMessage::text(prompt));
4930                    self.persist_turn_item(&req.thread_id, &turn_id, &item)
4931                        .await?;
4932                    transcript.push(item);
4933                    continue;
4934                }
4935                // Loop persistence nudge: in non-interactive/eval runs, when the
4936                // model returns a final message with no tool calls, gently prompt
4937                // it to keep going (bounded by `empty_tool_call_nudges`) before
4938                // ending the turn. Gated to non-interactive/eval so interactive
4939                // chat completions are never delayed, and only fires when the
4940                // model actually produced a final answer to re-examine.
4941                if !deadline_finalization_requested
4942                    && runtime_profile != RuntimeProfile::Interactive
4943                    && cfg.reliability.empty_tool_call_nudges > 0
4944                    && empty_tool_call_nudges_used < cfg.reliability.empty_tool_call_nudges
4945                    && (!assistant_text.trim().is_empty() || !phase_messages.is_empty())
4946                {
4947                    empty_tool_call_nudges_used += 1;
4948                    let item = TranscriptItem::UserMessage(UserMessage::text(
4949                        RELIABILITY_CONTINUATION_PROMPT.to_string(),
4950                    ));
4951                    self.persist_turn_item(&req.thread_id, &turn_id, &item)
4952                        .await?;
4953                    transcript.push(item);
4954                    continue;
4955                }
4956                if deadline_finalization_requested
4957                    && assistant_text.trim().is_empty()
4958                    && phase_messages.is_empty()
4959                {
4960                    assistant_text = format!(
4961                        "Deadline finalization completed without model text. {}",
4962                        turn_partial_result(&transcript)
4963                    );
4964                }
4965                final_assistant_text = assistant_text;
4966                final_provider_metadata = provider_metadata;
4967                exhausted_tool_rounds = false;
4968                break;
4969            }
4970
4971            if let Some(metadata) = provider_metadata {
4972                let had_provider_compaction =
4973                    crate::compaction::provider_metadata_has_compaction(&metadata);
4974                let item = TranscriptItem::ProviderMetadata(metadata);
4975                self.persist_turn_item(&req.thread_id, &turn_id, &item)
4976                    .await?;
4977                transcript.push(item);
4978                if had_provider_compaction {
4979                    // Server-side compaction replaced the prior window; drop
4980                    // pre-boundary items so subsequent tool rounds use the
4981                    // compact window as the next input.
4982                    transcript = crate::compaction::trim_to_last_compaction_boundary(transcript);
4983                    compacted_this_turn = true;
4984                }
4985            }
4986            eager_tools.drain().await?;
4987            for call in &tool_calls {
4988                if eager_tools.scheduled(&call.id) {
4989                    continue;
4990                }
4991                let tool_item = TranscriptItem::ToolCall(ToolCallRecord {
4992                    id: call.id.clone(),
4993                    name: call.name.clone(),
4994                    arguments: call.arguments.clone(),
4995                });
4996                self.persist_turn_item(&req.thread_id, &turn_id, &tool_item)
4997                    .await?;
4998                transcript.push(tool_item);
4999                self.persist_model_profile_segment(
5000                    &req.thread_id,
5001                    &turn_id,
5002                    model_profile.as_ref(),
5003                    &provider,
5004                    &model,
5005                    "tool_call",
5006                )
5007                .await?;
5008            }
5009            if let Some(deadline) = turn_deadline
5010                && deadline_expired(deadline)
5011            {
5012                self.fail_turn_due_to_deadline(&req.thread_id, &turn_id, deadline, &transcript)
5013                    .await?;
5014                return Ok(TurnRunOutcome::Stopped);
5015            }
5016            let mut results = eager_tools.take_results(&tool_calls);
5017            let remaining_calls = tool_calls
5018                .into_iter()
5019                .filter(|call| !eager_tools.scheduled(&call.id))
5020                .collect();
5021            results.extend(
5022                self.route_tool_calls(
5023                    &req.thread_id,
5024                    &turn_id,
5025                    remaining_calls,
5026                    parallel_tool_calls,
5027                    Some(workspace.as_str()),
5028                    turn_deadline,
5029                )
5030                .await?,
5031            );
5032            let reliability_limit = reliability.record_tool_results(
5033                &cfg.reliability,
5034                &results,
5035                runtime_profile == RuntimeProfile::Interactive,
5036            );
5037            for result in results {
5038                verification_gate.record_tool_result(&result);
5039                transcript.push(TranscriptItem::ToolResult(result));
5040                self.persist_model_profile_segment(
5041                    &req.thread_id,
5042                    &turn_id,
5043                    model_profile.as_ref(),
5044                    &provider,
5045                    &model,
5046                    "tool_result",
5047                )
5048                .await?;
5049            }
5050            if let Some(limit) = reliability_limit {
5051                if limit.decision == ReliabilityLimitDecision::RequestContinuation {
5052                    // Loop persistence: rather than ending the turn, reset the
5053                    // consecutive-failure counter, nudge the model to keep going,
5054                    // and continue the round loop. Bounded by the per-turn tool
5055                    // failure ceiling, `max_model_calls_per_turn`, and
5056                    // `MAX_TOOL_ROUNDS_PER_TURN`.
5057                    self.record_reliability_limit_continuation(
5058                        &req.thread_id,
5059                        &turn_id,
5060                        &provider,
5061                        &model,
5062                        limit,
5063                    )
5064                    .await;
5065                    reliability.reset_consecutive_failures();
5066                    let nudge = TranscriptItem::UserMessage(UserMessage::text(
5067                        RELIABILITY_CONTINUATION_PROMPT.to_string(),
5068                    ));
5069                    self.persist_turn_item(&req.thread_id, &turn_id, &nudge)
5070                        .await?;
5071                    transcript.push(nudge);
5072                } else {
5073                    self.fail_turn_due_to_reliability_limit(
5074                        &req.thread_id,
5075                        &turn_id,
5076                        &provider,
5077                        &model,
5078                        limit,
5079                        &transcript,
5080                    )
5081                    .await?;
5082                    return Ok(TurnRunOutcome::Stopped);
5083                }
5084            }
5085            let compaction_generation_before = self.compaction_generation(&req.thread_id);
5086            transcript = self
5087                .compact_transcript_if_needed(
5088                    &req.thread_id,
5089                    &turn_id,
5090                    &provider,
5091                    &model,
5092                    transcript,
5093                    self.compaction_options_for_turn(&req.thread_id, !compacted_this_turn),
5094                )
5095                .await?;
5096            compacted_this_turn |=
5097                self.compaction_generation(&req.thread_id) != compaction_generation_before;
5098        }
5099
5100        if exhausted_tool_rounds {
5101            let message =
5102                format!("tool call limit reached after {MAX_TOOL_ROUNDS_PER_TURN} rounds");
5103            self.persist_turn_item(
5104                &req.thread_id,
5105                &turn_id,
5106                &TranscriptItem::Error(ErrorRecord {
5107                    message: message.clone(),
5108                }),
5109            )
5110            .await?;
5111            self.emit(RoderEvent::TurnFailed(TurnFailed {
5112                thread_id: req.thread_id.clone(),
5113                turn_id: turn_id.clone(),
5114                error: message.clone(),
5115                error_kind: None,
5116                usage: None,
5117                timestamp: OffsetDateTime::now_utc(),
5118            }))
5119            .await;
5120            self.complete_team_member_turn_with_result(
5121                &req.thread_id,
5122                &turn_id,
5123                TeamMemberStatus::Failed,
5124                None,
5125                Some(message),
5126            )
5127            .await?;
5128            return Ok(TurnRunOutcome::Stopped);
5129        }
5130
5131        if let Some(metadata) = final_provider_metadata {
5132            self.persist_turn_item(
5133                &req.thread_id,
5134                &turn_id,
5135                &TranscriptItem::ProviderMetadata(metadata),
5136            )
5137            .await?;
5138        }
5139
5140        let turn_usage_tokens = turn_usage.total_tokens as i64;
5141        let completed_usage = (!turn_usage.is_empty()).then_some(turn_usage.clone());
5142        self.record_thread_usage_metadata(&req.thread_id, &turn_usage)
5143            .await?;
5144        self.goals
5145            .account_turn_usage(
5146                &req.thread_id,
5147                turn_usage_tokens,
5148                OffsetDateTime::now_utc() - turn_started_at,
5149            )
5150            .await?;
5151        let (cleanup, ownership) = self.await_provider_turn_cleanup(&turn_id).await;
5152        self.record_turn_lifecycle_with_ownership(
5153            req.thread_id.clone(),
5154            turn_id.clone(),
5155            TurnLifecycleState::Completed,
5156            cleanup,
5157            None,
5158            ownership,
5159        )
5160        .await;
5161        self.emit(RoderEvent::TurnCompleted(TurnCompleted {
5162            thread_id: req.thread_id.clone(),
5163            turn_id: turn_id.clone(),
5164            usage: completed_usage,
5165            finish_reason: turn_finish_reason,
5166            timestamp: OffsetDateTime::now_utc(),
5167        }))
5168        .await;
5169        self.complete_team_member_turn_with_result(
5170            &req.thread_id,
5171            &turn_id,
5172            TeamMemberStatus::Completed,
5173            (!final_assistant_text.is_empty()).then_some(final_assistant_text),
5174            None,
5175        )
5176        .await?;
5177        Ok(TurnRunOutcome::Completed)
5178    }
5179
5180    async fn route_tool_calls(
5181        self: &Arc<Self>,
5182        thread_id: &ThreadId,
5183        turn_id: &TurnId,
5184        calls: Vec<ToolCallCompleted>,
5185        parallel: bool,
5186        workspace: Option<&str>,
5187        deadline: Option<OffsetDateTime>,
5188    ) -> anyhow::Result<Vec<ToolResultRecord>> {
5189        // Enforce the agent_swarm exclusivity rule (roadmap 104, Task 2):
5190        // `agent_swarm` must be the only tool call in a model response. A mixed
5191        // or multi-swarm batch is denied wholesale with actionable retry text so
5192        // the model re-issues `agent_swarm` by itself, and every tool_call_id
5193        // still gets a response (keeping the chat-completions transcript valid).
5194        if let Some(violation) = roder_api::subagents::agent_swarm_batch_violation(
5195            calls.iter().map(|call| call.name.as_str()),
5196        ) {
5197            let message = violation.deny_message();
5198            return Ok(calls
5199                .into_iter()
5200                .map(|call| ToolResultRecord {
5201                    id: call.id,
5202                    name: Some(call.name),
5203                    result: message.clone(),
5204                    display_payload: None,
5205                    is_error: true,
5206                })
5207                .collect());
5208        }
5209        // Emit swarm lifecycle events on the bus (roadmap 104, Task 1) so any
5210        // app-server/SDK/TUI client can observe a swarm as a whole; per-child
5211        // progress flows through the existing Subagent* trace events.
5212        let swarm_call = (calls.len() == 1
5213            && calls[0].name == roder_api::subagents::AGENT_SWARM_TOOL_NAME)
5214            .then(|| (calls[0].id.clone(), calls[0].arguments.clone()));
5215        if let Some((tool_id, args)) = &swarm_call {
5216            self.emit(RoderEvent::AgentSwarmStarted(
5217                roder_api::subagents::AgentSwarmStarted {
5218                    thread_id: thread_id.clone(),
5219                    turn_id: turn_id.clone(),
5220                    tool_id: tool_id.clone(),
5221                    child_count: agent_swarm_child_count(args),
5222                    timestamp: OffsetDateTime::now_utc(),
5223                },
5224            ))
5225            .await;
5226        }
5227
5228        let force_sequential = calls
5229            .iter()
5230            .any(|call| crate::agent_control_tools::is_agent_control_tool(&call.name));
5231        let results =
5232            if parallel && !force_sequential {
5233                try_join_all(calls.into_iter().map(|call| {
5234                    self.route_tool_call(thread_id, turn_id, call, workspace, deadline)
5235                }))
5236                .await
5237            } else {
5238                let mut results = Vec::with_capacity(calls.len());
5239                for call in calls {
5240                    results.push(
5241                        self.route_tool_call(thread_id, turn_id, call, workspace, deadline)
5242                            .await?,
5243                    );
5244                }
5245                Ok(results)
5246            }?;
5247
5248        if let Some((tool_id, _)) = &swarm_call
5249            && let Some(result) = results.iter().find(|result| &result.id == tool_id)
5250            && let Some((completed, failed, aborted)) = parse_swarm_counts(&result.result)
5251        {
5252            self.emit(RoderEvent::AgentSwarmCompleted(
5253                roder_api::subagents::AgentSwarmCompleted {
5254                    thread_id: thread_id.clone(),
5255                    turn_id: turn_id.clone(),
5256                    tool_id: tool_id.clone(),
5257                    completed,
5258                    failed,
5259                    aborted,
5260                    timestamp: OffsetDateTime::now_utc(),
5261                },
5262            ))
5263            .await;
5264        }
5265
5266        Ok(results)
5267    }
5268
5269    /// On provider context/prompt-length failures, force-compact (and if needed
5270    /// strip the last bulky item) then retry the current tool round instead of
5271    /// failing the turn. Bounded so a permanently oversized suffix still stops.
5272    async fn try_recover_from_context_limit(
5273        &self,
5274        thread_id: &ThreadId,
5275        turn_id: &TurnId,
5276        provider: &str,
5277        model: &str,
5278        error: &str,
5279        transcript: &mut Vec<TranscriptItem>,
5280        compacted_this_turn: &mut bool,
5281        recovery_attempts: &mut u32,
5282    ) -> anyhow::Result<bool> {
5283        const MAX_CONTEXT_LIMIT_RECOVERY_ATTEMPTS: u32 = 2;
5284        if !crate::compaction::is_context_limit_failure_message(error) {
5285            return Ok(false);
5286        }
5287        if *recovery_attempts >= MAX_CONTEXT_LIMIT_RECOVERY_ATTEMPTS {
5288            return Ok(false);
5289        }
5290        *recovery_attempts = recovery_attempts.saturating_add(1);
5291
5292        let error_item = TranscriptItem::Error(ErrorRecord {
5293            message: error.to_string(),
5294        });
5295        self.persist_turn_item(thread_id, turn_id, &error_item)
5296            .await?;
5297        transcript.push(error_item);
5298
5299        // After the first failed recovery, drop the item ahead of the latest
5300        // user/error so a second compact can succeed when the suffix itself is
5301        // still oversized (e.g. a huge tool result right before the prompt).
5302        if *recovery_attempts > 1 {
5303            *transcript =
5304                crate::compaction::strip_last_message_before_error(std::mem::take(transcript));
5305        }
5306
5307        let force_options = crate::compaction::CompactionOptions {
5308            allow_repeat: true,
5309            force: true,
5310            hysteresis_baseline: None,
5311            preserve_hint: Some(
5312                "Recover from a provider context/prompt-length limit; preserve the active user goal and recent tool outcomes."
5313                    .to_string(),
5314            ),
5315        };
5316        let before_len = transcript.len();
5317        let before_tokens = crate::compaction::estimate_prompt_tokens(transcript);
5318        *transcript = self
5319            .compact_transcript_if_needed(
5320                thread_id,
5321                turn_id,
5322                provider,
5323                model,
5324                std::mem::take(transcript),
5325                force_options,
5326            )
5327            .await?;
5328        let after_tokens = crate::compaction::estimate_prompt_tokens(transcript);
5329        *compacted_this_turn = *compacted_this_turn
5330            || transcript
5331                .iter()
5332                .any(crate::compaction::is_compaction_boundary);
5333
5334        self.emit(RoderEvent::ReliabilityRetryRecorded(
5335            ReliabilityRetryRecorded {
5336                context: ReliabilityContext {
5337                    thread_id: thread_id.clone(),
5338                    turn_id: turn_id.clone(),
5339                    provider: Some(provider.to_string()),
5340                    model: Some(model.to_string()),
5341                    ..ReliabilityContext::default()
5342                },
5343                error_class: ReliabilityErrorClass::ProviderError,
5344                decision: ReliabilityRetryDecision::Retry,
5345                attempt: *recovery_attempts,
5346                max_attempts: MAX_CONTEXT_LIMIT_RECOVERY_ATTEMPTS,
5347                delay_ms: Some(0),
5348                details: ReliabilityDetails::redacted(format!(
5349                    "context_limit_recovery: compact {before_tokens}->{after_tokens} tokens (items {before_len}->{}); {error}",
5350                    transcript.len()
5351                )),
5352                timestamp: OffsetDateTime::now_utc(),
5353            },
5354        ))
5355        .await;
5356
5357        // If force-compact did not shrink anything and we still have budget,
5358        // strip once more so the next round is not identical.
5359        if after_tokens >= before_tokens && *recovery_attempts < MAX_CONTEXT_LIMIT_RECOVERY_ATTEMPTS
5360        {
5361            *transcript =
5362                crate::compaction::strip_last_message_before_error(std::mem::take(transcript));
5363        }
5364        Ok(true)
5365    }
5366
5367    async fn fail_turn_with_error(
5368        &self,
5369        thread_id: &ThreadId,
5370        turn_id: &TurnId,
5371        message: String,
5372    ) -> anyhow::Result<()> {
5373        self.persist_turn_item(
5374            thread_id,
5375            turn_id,
5376            &TranscriptItem::Error(ErrorRecord {
5377                message: message.clone(),
5378            }),
5379        )
5380        .await?;
5381        self.emit(RoderEvent::TurnFailed(TurnFailed {
5382            thread_id: thread_id.clone(),
5383            turn_id: turn_id.clone(),
5384            error: message.clone(),
5385            error_kind: None,
5386            usage: None,
5387            timestamp: OffsetDateTime::now_utc(),
5388        }))
5389        .await;
5390        self.complete_team_member_turn_with_result(
5391            thread_id,
5392            turn_id,
5393            TeamMemberStatus::Failed,
5394            None,
5395            Some(message),
5396        )
5397        .await?;
5398        Ok(())
5399    }
5400
5401    async fn fail_turn_due_to_deadline(
5402        &self,
5403        thread_id: &ThreadId,
5404        turn_id: &TurnId,
5405        deadline: OffsetDateTime,
5406        transcript: &[TranscriptItem],
5407    ) -> anyhow::Result<()> {
5408        let partial_result = turn_partial_result(transcript);
5409        self.emit(RoderEvent::TurnPartialResult(TurnPartialResult {
5410            thread_id: thread_id.clone(),
5411            turn_id: turn_id.clone(),
5412            summary: partial_result.clone(),
5413            timestamp: OffsetDateTime::now_utc(),
5414        }))
5415        .await;
5416        self.emit(RoderEvent::TurnDeadlineExceeded(TurnDeadlineExceeded {
5417            thread_id: thread_id.clone(),
5418            turn_id: turn_id.clone(),
5419            deadline,
5420            partial_result: partial_result.clone(),
5421            timestamp: OffsetDateTime::now_utc(),
5422        }))
5423        .await;
5424        let message = "turn deadline expired".to_string();
5425        self.persist_turn_item(
5426            thread_id,
5427            turn_id,
5428            &TranscriptItem::Error(ErrorRecord {
5429                message: format!("{message}: {partial_result}"),
5430            }),
5431        )
5432        .await?;
5433        self.emit(RoderEvent::TurnFailed(TurnFailed {
5434            thread_id: thread_id.clone(),
5435            turn_id: turn_id.clone(),
5436            error: message.clone(),
5437            error_kind: Some("deadline_timeout".to_string()),
5438            usage: None,
5439            timestamp: OffsetDateTime::now_utc(),
5440        }))
5441        .await;
5442        self.complete_team_member_turn_with_result(
5443            thread_id,
5444            turn_id,
5445            TeamMemberStatus::Failed,
5446            None,
5447            Some(format!("{message}: {partial_result}")),
5448        )
5449        .await?;
5450        Ok(())
5451    }
5452
5453    async fn start_deadline_finalization(
5454        &self,
5455        thread_id: &ThreadId,
5456        turn_id: &TurnId,
5457        transcript: &mut Vec<TranscriptItem>,
5458        remaining_seconds: u64,
5459    ) -> anyhow::Result<()> {
5460        let item = TranscriptItem::UserMessage(crate::deadline_policy::finalization_message(
5461            remaining_seconds,
5462        ));
5463        self.persist_turn_item(thread_id, turn_id, &item).await?;
5464        transcript.push(item);
5465        self.emit(RoderEvent::TurnPartialResult(TurnPartialResult {
5466            thread_id: thread_id.clone(),
5467            turn_id: turn_id.clone(),
5468            summary: turn_partial_result(transcript),
5469            timestamp: OffsetDateTime::now_utc(),
5470        }))
5471        .await;
5472        Ok(())
5473    }
5474
5475    /// Emits the reliability-limit event for a continuation (loop persistence)
5476    /// without failing the turn. The caller resets the failure counter and
5477    /// injects a nudge so the round loop keeps going.
5478    async fn record_reliability_limit_continuation(
5479        &self,
5480        thread_id: &ThreadId,
5481        turn_id: &TurnId,
5482        provider: &str,
5483        model: &str,
5484        limit: ReliabilityLimitHit,
5485    ) {
5486        self.emit(RoderEvent::ReliabilityLimitRecorded(
5487            ReliabilityLimitRecorded {
5488                context: ReliabilityContext {
5489                    thread_id: thread_id.clone(),
5490                    turn_id: turn_id.clone(),
5491                    tool_id: None,
5492                    tool_name: None,
5493                    provider: Some(provider.to_string()),
5494                    model: Some(model.to_string()),
5495                },
5496                error_class: limit.error_class,
5497                limit_kind: limit.limit_kind,
5498                decision: limit.decision,
5499                current: limit.current,
5500                limit: limit.limit,
5501                details: ReliabilityDetails::redacted(&limit.message),
5502                timestamp: OffsetDateTime::now_utc(),
5503            },
5504        ))
5505        .await;
5506    }
5507
5508    async fn fail_turn_due_to_reliability_limit(
5509        &self,
5510        thread_id: &ThreadId,
5511        turn_id: &TurnId,
5512        provider: &str,
5513        model: &str,
5514        limit: ReliabilityLimitHit,
5515        transcript: &[TranscriptItem],
5516    ) -> anyhow::Result<()> {
5517        self.emit(RoderEvent::ReliabilityLimitRecorded(
5518            ReliabilityLimitRecorded {
5519                context: ReliabilityContext {
5520                    thread_id: thread_id.clone(),
5521                    turn_id: turn_id.clone(),
5522                    tool_id: None,
5523                    tool_name: None,
5524                    provider: Some(provider.to_string()),
5525                    model: Some(model.to_string()),
5526                },
5527                error_class: limit.error_class,
5528                limit_kind: limit.limit_kind,
5529                decision: limit.decision,
5530                current: limit.current,
5531                limit: limit.limit,
5532                details: ReliabilityDetails::redacted(&limit.message),
5533                timestamp: OffsetDateTime::now_utc(),
5534            },
5535        ))
5536        .await;
5537        let partial_result = turn_partial_result(transcript);
5538        self.emit(RoderEvent::TurnPartialResult(TurnPartialResult {
5539            thread_id: thread_id.clone(),
5540            turn_id: turn_id.clone(),
5541            summary: partial_result.clone(),
5542            timestamp: OffsetDateTime::now_utc(),
5543        }))
5544        .await;
5545        let message = format!("reliability limit reached: {}", limit.message);
5546        self.persist_turn_item(
5547            thread_id,
5548            turn_id,
5549            &TranscriptItem::Error(ErrorRecord {
5550                message: format!("{message}: {partial_result}"),
5551            }),
5552        )
5553        .await?;
5554        self.emit(RoderEvent::TurnFailed(TurnFailed {
5555            thread_id: thread_id.clone(),
5556            turn_id: turn_id.clone(),
5557            error: message.clone(),
5558            error_kind: Some("reliability_limit".to_string()),
5559            usage: None,
5560            timestamp: OffsetDateTime::now_utc(),
5561        }))
5562        .await;
5563        self.complete_team_member_turn_with_result(
5564            thread_id,
5565            turn_id,
5566            TeamMemberStatus::Failed,
5567            None,
5568            Some(format!("{message}: {partial_result}")),
5569        )
5570        .await?;
5571        Ok(())
5572    }
5573
5574    async fn append_steers(
5575        &self,
5576        req: &StartTurnRequest,
5577        turn_id: &TurnId,
5578        transcript: &mut Vec<TranscriptItem>,
5579        steers: Vec<QueuedTurnSteer>,
5580    ) -> anyhow::Result<()> {
5581        for queued in steers {
5582            let mut steer = queued.message;
5583            steer.text = steer.text.trim().to_string();
5584            if steer.text.is_empty() && steer.images.is_empty() {
5585                continue;
5586            }
5587            let item = TranscriptItem::UserMessage(steer);
5588            self.persist_turn_item(&req.thread_id, turn_id, &item)
5589                .await?;
5590            transcript.push(item);
5591            if let Some(ack) = queued.mailbox_ack {
5592                self.teams
5593                    .mark_mailbox_messages_delivered(&ack.team_id, turn_id, &ack.message_ids)
5594                    .await?;
5595            }
5596        }
5597        Ok(())
5598    }
5599
5600    async fn persist_model_profile_segment(
5601        &self,
5602        thread_id: &ThreadId,
5603        turn_id: &TurnId,
5604        profile: Option<&ModelHarnessProfile>,
5605        provider: &str,
5606        model: &str,
5607        segment: &str,
5608    ) -> anyhow::Result<()> {
5609        let item = TranscriptItem::ProviderMetadata(model_profile_segment_metadata(
5610            profile, provider, model, segment,
5611        ));
5612        self.persist_turn_item(thread_id, turn_id, &item).await
5613    }
5614
5615    fn task_ledger_tool_specs(
5616        &self,
5617        profile: Option<&ModelHarnessProfile>,
5618    ) -> Vec<roder_api::tools::ToolSpec> {
5619        self.tool_registry
5620            .get(TASK_LEDGER_TOOL_NAME)
5621            .map(|tool| {
5622                tool.spec()
5623                    .normalized_for_model_profile(schema_policy_for_model(profile))
5624            })
5625            .into_iter()
5626            .collect()
5627    }
5628
5629    pub(crate) fn engine_for(&self, provider: &str) -> anyhow::Result<Arc<dyn InferenceEngine>> {
5630        self.registry
5631            .inference_engine(provider)
5632            .or_else(|| {
5633                self.registry
5634                    .default_inference_engine()
5635                    .filter(|engine| provider.is_empty() || engine.id() == provider)
5636            })
5637            .ok_or_else(|| anyhow::anyhow!("inference provider {provider:?} is not registered"))
5638    }
5639
5640    pub async fn emit(&self, event: RoderEvent) -> EventEnvelope {
5641        self.dispatch_local_hook_lifecycle(&event).await;
5642        if let Some(record) = Self::lifecycle_record_for_event(&event) {
5643            let _ = self.persist_turn_lifecycle_record(&record).await;
5644        }
5645        let envelope = self.bus.emit(event);
5646        if let (Some(store), Some(thread_id)) = (&self.thread_store, envelope.thread_id.as_ref())
5647            && should_persist_thread_event(thread_id)
5648        {
5649            let _ = store.append_event(thread_id, &envelope).await;
5650        }
5651        // Registered event sinks (e.g. process extensions) receive the
5652        // persisted envelope through bounded per-sink queues; a slow sink
5653        // never blocks emit or turn progress.
5654        let dispatcher = self
5655            .event_sink_dispatcher
5656            .get_or_init(|| async {
5657                crate::event_sink_dispatch::EventSinkDispatcher::start(
5658                    &self.registry.event_sinks,
5659                    self.bus.clone(),
5660                )
5661            })
5662            .await;
5663        if !dispatcher.is_empty() {
5664            dispatcher.dispatch(&envelope, &self.bus);
5665        }
5666        envelope
5667    }
5668
5669    async fn dispatch_local_hook_lifecycle(&self, event: &RoderEvent) {
5670        // SessionStart belongs to the session, not the turn: firing it from
5671        // TurnStarted re-ran setup hooks on every user message. ThreadCreated
5672        // and ThreadLoaded carry no turn id, so they borrow a synthetic one the
5673        // way the SessionEnd path does.
5674        let session_turn_id = SESSION_HOOK_TURN_ID.to_string();
5675        let (thread_id, turn_id, name, matcher, input) = match event {
5676            RoderEvent::ThreadCreated(event) => (
5677                &event.thread_id,
5678                &session_turn_id,
5679                "SessionStart",
5680                Some("startup"),
5681                serde_json::json!({"source":"startup"}),
5682            ),
5683            RoderEvent::ThreadLoaded(event) => (
5684                &event.thread_id,
5685                &session_turn_id,
5686                "SessionStart",
5687                Some("resume"),
5688                serde_json::json!({"source":"resume"}),
5689            ),
5690            RoderEvent::TurnCompleted(event) => (
5691                &event.thread_id,
5692                &event.turn_id,
5693                "Stop",
5694                None,
5695                serde_json::json!({"finishReason":event.finish_reason}),
5696            ),
5697            RoderEvent::TurnFailed(event) => (
5698                &event.thread_id,
5699                &event.turn_id,
5700                "Stop",
5701                None,
5702                serde_json::json!({"error":event.error}),
5703            ),
5704            RoderEvent::ContextCompactionStarted(event) => (
5705                &event.thread_id,
5706                &event.turn_id,
5707                "PreCompact",
5708                None,
5709                serde_json::json!({"originalItemCount":event.original_item_count,"originalEstimatedTokens":event.original_estimated_tokens}),
5710            ),
5711            RoderEvent::ContextCompactionRecorded(event) => (
5712                &event.thread_id,
5713                &event.turn_id,
5714                "PostCompact",
5715                None,
5716                serde_json::json!({"compactedItemCount":event.compacted_item_count,"compactedEstimatedTokens":event.compacted_estimated_tokens}),
5717            ),
5718            RoderEvent::SubagentStarted(event) => (
5719                &event.parent_thread_id,
5720                &event.parent_turn_id,
5721                "SubagentStart",
5722                Some(event.agent_type.as_str()),
5723                serde_json::json!({"subagentThreadId":event.thread_id,"agentType":event.agent_type,"description":event.description}),
5724            ),
5725            RoderEvent::SubagentCompleted(event) => (
5726                &event.parent_thread_id,
5727                &event.parent_turn_id,
5728                "SubagentStop",
5729                Some(event.agent_type.as_str()),
5730                serde_json::json!({"subagentThreadId":event.thread_id,"agentType":event.agent_type,"exitReason":event.exit_reason}),
5731            ),
5732            RoderEvent::SubagentFailed(event) => (
5733                &event.parent_thread_id,
5734                &event.parent_turn_id,
5735                "SubagentStop",
5736                Some(event.agent_type.as_str()),
5737                serde_json::json!({"subagentThreadId":event.thread_id,"agentType":event.agent_type,"error":event.error}),
5738            ),
5739            _ => return,
5740        };
5741        // One SessionStart per session, whichever event opened it.
5742        if name == "SessionStart" && !self.claim_session_hook_start(thread_id).await {
5743            return;
5744        }
5745        let workspace = self.config.read().await.workspace.clone();
5746        crate::hooks::run_lifecycle(
5747            self,
5748            thread_id,
5749            turn_id,
5750            workspace.as_deref(),
5751            name,
5752            matcher,
5753            input,
5754        )
5755        .await;
5756    }
5757
5758    /// Claim the single `SessionStart` dispatch for a thread, returning whether
5759    /// this caller won it.
5760    pub(crate) async fn claim_session_hook_start(&self, thread_id: &ThreadId) -> bool {
5761        self.session_hook_started
5762            .write()
5763            .await
5764            .insert(thread_id.clone())
5765    }
5766
5767    /// Records a projected thread item event and persists it through the configured thread store.
5768    ///
5769    /// App-server protocol notification bridges currently call this after translating runtime
5770    /// events into item events. Headless runtime consumers that subscribe to `RoderEvent` directly
5771    /// must make the same call if they need the derived item event stream persisted.
5772    pub async fn record_thread_item_event_kind(
5773        &self,
5774        thread_id: &ThreadId,
5775        turn_id: &TurnId,
5776        timestamp: OffsetDateTime,
5777        kind: ThreadItemEventKind,
5778    ) -> anyhow::Result<ThreadItemEvent> {
5779        let seq = self.next_thread_item_event_seq(thread_id).await?;
5780        let item_event = ThreadItemEvent {
5781            seq,
5782            event_id: format!("{turn_id}-item-event-{seq}"),
5783            thread_id: thread_id.clone(),
5784            turn_id: turn_id.clone(),
5785            timestamp,
5786            event: kind,
5787        };
5788        if let Some(store) = &self.thread_store {
5789            store.append_item_event(thread_id, &item_event).await?;
5790        }
5791        self.remember_thread_item_event(&item_event).await?;
5792        Ok(item_event)
5793    }
5794
5795    async fn next_thread_item_event_seq(&self, thread_id: &ThreadId) -> anyhow::Result<u64> {
5796        self.ensure_thread_item_cache(thread_id).await?;
5797        Ok(self
5798            .thread_item_cache
5799            .lock()
5800            .await
5801            .next_item_event_seq(thread_id))
5802    }
5803
5804    pub async fn thread_item_exists(
5805        &self,
5806        thread_id: &ThreadId,
5807        turn_id: &TurnId,
5808        item_id: &str,
5809    ) -> anyhow::Result<bool> {
5810        self.ensure_thread_item_cache(thread_id).await?;
5811        Ok(self
5812            .thread_item_cache
5813            .lock()
5814            .await
5815            .thread_item_exists(thread_id, turn_id, item_id))
5816    }
5817
5818    pub async fn current_reasoning_item_id(
5819        &self,
5820        thread_id: &ThreadId,
5821        turn_id: &TurnId,
5822    ) -> anyhow::Result<Option<String>> {
5823        self.ensure_thread_item_cache(thread_id).await?;
5824        Ok(self
5825            .thread_item_cache
5826            .lock()
5827            .await
5828            .current_reasoning_item_id(thread_id, turn_id))
5829    }
5830
5831    async fn remember_thread_item_event(&self, item_event: &ThreadItemEvent) -> anyhow::Result<()> {
5832        self.ensure_thread_item_cache(&item_event.thread_id).await?;
5833        self.thread_item_cache
5834            .lock()
5835            .await
5836            .remember_item_event(item_event);
5837        Ok(())
5838    }
5839
5840    pub async fn latest_transcript_item_index(
5841        &self,
5842        thread_id: &ThreadId,
5843        turn_id: &TurnId,
5844    ) -> anyhow::Result<Option<usize>> {
5845        self.ensure_thread_item_cache(thread_id).await?;
5846        Ok(self
5847            .thread_item_cache
5848            .lock()
5849            .await
5850            .latest_transcript_item_index(thread_id, turn_id))
5851    }
5852
5853    async fn next_transcript_item_index(
5854        &self,
5855        thread_id: &ThreadId,
5856        turn_id: &TurnId,
5857    ) -> anyhow::Result<usize> {
5858        self.ensure_thread_item_cache(thread_id).await?;
5859        Ok(self
5860            .thread_item_cache
5861            .lock()
5862            .await
5863            .next_transcript_item_index(thread_id, turn_id))
5864    }
5865
5866    async fn remember_transcript_item_index(
5867        &self,
5868        thread_id: &ThreadId,
5869        turn_id: &TurnId,
5870        item_index: usize,
5871    ) -> anyhow::Result<()> {
5872        self.ensure_thread_item_cache(thread_id).await?;
5873        self.thread_item_cache
5874            .lock()
5875            .await
5876            .remember_transcript_item_index(thread_id, turn_id, item_index);
5877        Ok(())
5878    }
5879
5880    async fn ensure_thread_item_cache(&self, thread_id: &ThreadId) -> anyhow::Result<()> {
5881        if self
5882            .thread_item_cache
5883            .lock()
5884            .await
5885            .contains_thread(thread_id)
5886        {
5887            return Ok(());
5888        }
5889
5890        let snapshot = if let Some(store) = &self.thread_store {
5891            store.load_thread(thread_id).await?
5892        } else {
5893            None
5894        };
5895        self.thread_item_cache.lock().await.ensure_thread(
5896            thread_id,
5897            ThreadItemCacheEntry::from_snapshot(snapshot.as_ref()),
5898        );
5899        Ok(())
5900    }
5901
5902    pub(crate) async fn persist_turn_item(
5903        &self,
5904        thread_id: &ThreadId,
5905        turn_id: &TurnId,
5906        item: &TranscriptItem,
5907    ) -> anyhow::Result<()> {
5908        let item_index = self.next_transcript_item_index(thread_id, turn_id).await?;
5909        let timestamp = OffsetDateTime::now_utc();
5910        self.emit(RoderEvent::TranscriptItemAppended(TranscriptItemAppended {
5911            thread_id: thread_id.clone(),
5912            turn_id: turn_id.clone(),
5913            item_type: match item {
5914                TranscriptItem::UserMessage(_) => "user_message",
5915                TranscriptItem::AssistantMessage(_) => "assistant_message",
5916                TranscriptItem::ReasoningSummary(_) => "reasoning_summary",
5917                TranscriptItem::ToolCall(_) => "tool_call",
5918                TranscriptItem::ToolResult(_) => "tool_result",
5919                TranscriptItem::FileChange(_) => "file_change",
5920                TranscriptItem::ContextCompaction(_) => "context_compaction",
5921                TranscriptItem::Error(_) => "error",
5922                TranscriptItem::ProviderMetadata(_) => "provider_metadata",
5923            }
5924            .to_string(),
5925            item_index: Some(item_index),
5926            item: Some(item.clone()),
5927            timestamp,
5928        }))
5929        .await;
5930        self.remember_transcript_item_index(thread_id, turn_id, item_index)
5931            .await?;
5932        Ok(())
5933    }
5934}
5935
5936fn transcript_has_images(transcript: &[TranscriptItem]) -> bool {
5937    transcript.iter().any(|item| {
5938        matches!(
5939            item,
5940            TranscriptItem::UserMessage(message) if !message.images.is_empty()
5941        )
5942    })
5943}
5944
5945fn transcript_has_task_ledger(transcript: &[TranscriptItem]) -> bool {
5946    transcript.iter().any(|item| {
5947        matches!(
5948            item,
5949            TranscriptItem::ToolResult(result)
5950                if result.name.as_deref() == Some(TASK_LEDGER_TOOL_NAME) && !result.is_error
5951        )
5952    })
5953}
5954
5955fn task_ledger_completion_prompt(transcript: &[TranscriptItem]) -> Option<String> {
5956    let latest = transcript.iter().rev().find_map(|item| match item {
5957        TranscriptItem::ToolResult(result)
5958            if result.name.as_deref() == Some(TASK_LEDGER_TOOL_NAME) && !result.is_error =>
5959        {
5960            Some(result.result.as_str())
5961        }
5962        _ => None,
5963    })?;
5964    if !task_ledger_has_open_items(latest) {
5965        return None;
5966    }
5967
5968    let mut ledger = latest.chars().take(1500).collect::<String>();
5969    if latest.chars().nth(1500).is_some() {
5970        ledger.push_str("...");
5971    }
5972    Some(format!(
5973        "Task Ledger Completion Required: the latest task ledger still has pending or in-progress items. Do not provide a final answer yet. Use tools to complete the remaining scoreable work, create or update any required output files, then call `{TASK_LEDGER_TOOL_NAME}` with every task completed and evidence before finalizing.\n\nLatest ledger:\n{ledger}"
5974    ))
5975}
5976
5977fn task_ledger_deadline_completion_prompt(
5978    remaining_seconds: u64,
5979    reserve_seconds: u64,
5980    completion_prompt: &str,
5981) -> String {
5982    format!(
5983        "Eval deadline scoreable completion: {remaining_seconds} seconds remain in the {reserve_seconds}-second finalization reserve. Do not browse, search, or start slow work. Use the available tools now to create or update the required scoreable output files, run only a quick local check if needed, then update the task ledger to completed before finalizing.\n\n{completion_prompt}"
5984    )
5985}
5986
5987fn task_ledger_scoreable_checkpoint_prompt(
5988    remaining_seconds: u64,
5989    completion_prompt: &str,
5990) -> String {
5991    format!(
5992        "Scoreable Output Checkpoint: {remaining_seconds} seconds remain before the eval deadline. Before any further research, browsing, or long commands, use tools now to ensure the required output file(s) exist with the best evidence-backed answer, even if provisional. If a scoreable file already exists, read it and preserve that candidate unless you have stronger task-specific evidence for a replacement. Do not overwrite a plausible dated, historical, or local-evidence candidate with a current live-page, partial-coverage, or weaker guess merely to refresh the checkpoint. You may continue refining afterward, but do not apologize or finalize until the scoreable file exists and the task ledger is updated.\n\n{completion_prompt}"
5993    )
5994}
5995
5996fn task_ledger_has_open_items(ledger: &str) -> bool {
5997    ledger.lines().any(|line| {
5998        let line = line.trim_start();
5999        line.starts_with("- pending:") || line.starts_with("- in_progress:")
6000    })
6001}
6002
6003fn turn_deadline_for_config(cfg: &RuntimeConfig) -> Option<OffsetDateTime> {
6004    cfg.turn_deadline_seconds
6005        .filter(|seconds| *seconds > 0)
6006        .map(|seconds| OffsetDateTime::now_utc() + Duration::seconds(seconds as i64))
6007}
6008
6009fn deadline_expired(deadline: OffsetDateTime) -> bool {
6010    OffsetDateTime::now_utc() >= deadline
6011}
6012
6013pub(crate) fn deadline_remaining_seconds(deadline: Option<OffsetDateTime>) -> Option<u64> {
6014    let deadline = deadline?;
6015    if deadline <= OffsetDateTime::now_utc() {
6016        return Some(0);
6017    }
6018    Some(
6019        (deadline - OffsetDateTime::now_utc())
6020            .unsigned_abs()
6021            .as_secs()
6022            .max(1),
6023    )
6024}
6025
6026fn deadline_instant(deadline: OffsetDateTime) -> tokio::time::Instant {
6027    let now = OffsetDateTime::now_utc();
6028    if deadline <= now {
6029        return tokio::time::Instant::now();
6030    }
6031    tokio::time::Instant::now() + (deadline - now).unsigned_abs()
6032}
6033
6034fn inference_timeout_deadline(
6035    deadline: Option<OffsetDateTime>,
6036    runtime_profile: RuntimeProfile,
6037    task_ledger_required: bool,
6038    reserve_seconds: u64,
6039    finalization_requested: bool,
6040    task_ledger_scoreable_checkpoints: u8,
6041    transcript: &[TranscriptItem],
6042) -> Option<(OffsetDateTime, InferenceTimeoutAction)> {
6043    let deadline = deadline?;
6044    if runtime_profile == RuntimeProfile::Eval
6045        && task_ledger_required
6046        && !finalization_requested
6047        && task_ledger_scoreable_checkpoints < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
6048        && TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS > reserve_seconds
6049        && task_ledger_completion_prompt(transcript).is_some()
6050    {
6051        let checkpoint_deadline =
6052            deadline - Duration::seconds(TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS as i64);
6053        if checkpoint_deadline > OffsetDateTime::now_utc() {
6054            return Some((
6055                checkpoint_deadline,
6056                InferenceTimeoutAction::ScoreableCheckpoint,
6057            ));
6058        }
6059    }
6060    if runtime_profile == RuntimeProfile::Eval && !finalization_requested {
6061        return Some((
6062            deadline - Duration::seconds(reserve_seconds as i64),
6063            InferenceTimeoutAction::Finalization,
6064        ));
6065    }
6066    Some((deadline, InferenceTimeoutAction::Finalization))
6067}
6068
6069fn turn_partial_result(transcript: &[TranscriptItem]) -> String {
6070    let tool_results = transcript
6071        .iter()
6072        .filter(|item| matches!(item, TranscriptItem::ToolResult(_)))
6073        .count();
6074    let assistant_messages = transcript
6075        .iter()
6076        .filter(|item| matches!(item, TranscriptItem::AssistantMessage(_)))
6077        .count();
6078    format!(
6079        "partial turn state: {} transcript items, {assistant_messages} assistant messages, {tool_results} tool results",
6080        transcript.len()
6081    )
6082}
6083
6084fn reasoning_for_model(cfg: &RuntimeConfig, model: &str) -> ReasoningConfig {
6085    let level = effective_reasoning_for_model(cfg, model);
6086    match level.as_str() {
6087        "" | REASONING_NONE => ReasoningConfig::default(),
6088        level => ReasoningConfig {
6089            enabled: true,
6090            level: Some(level.to_string()),
6091        },
6092    }
6093}
6094
6095fn server_side_compaction_threshold(cfg: &RuntimeConfig, model: &str) -> Option<u32> {
6096    let entry = lookup_model(model)?;
6097    if !entry.supports_compaction {
6098        return None;
6099    }
6100    cfg.auto_compact_token_limit
6101        .or_else(|| {
6102            model_profile_for_model(cfg, model).and_then(|profile| profile.auto_compact_token_limit)
6103        })
6104        .or(Some(entry.auto_compact_token_limit))
6105        .filter(|threshold| *threshold > 0)
6106}
6107
6108pub(crate) fn tool_search_for_provider_model(
6109    cfg: &RuntimeConfig,
6110    provider: &str,
6111    model: &str,
6112) -> ToolSearchConfig {
6113    let mut resolved = cfg.tool_search.clone();
6114    if let Some(provider_config) = cfg.provider_tool_search.get(provider) {
6115        provider_config.apply_to(&mut resolved);
6116    }
6117    if let Some(model_config) = cfg.model_tool_search.get(model) {
6118        model_config.apply_to(&mut resolved);
6119    }
6120    resolved
6121}
6122
6123fn parallel_tool_calls_for_model(cfg: &RuntimeConfig, model: &str) -> bool {
6124    cfg.model_parallel_tool_calls
6125        .get(model)
6126        .copied()
6127        .or_else(|| {
6128            model_profile_for_model(cfg, model).and_then(|profile| profile.parallel_tool_calls)
6129        })
6130        .unwrap_or(true)
6131}
6132
6133/// Count the children an `agent_swarm` call will launch from its arguments
6134/// (item-based spawns plus resumes). Lenient: returns 0 on malformed input.
6135fn agent_swarm_child_count(arguments: &str) -> usize {
6136    serde_json::from_str::<roder_api::subagents::AgentSwarmRequest>(arguments)
6137        .map(|request| request.items.len() + request.resume_agent_ids.len())
6138        .unwrap_or(0)
6139}
6140
6141/// Parse `completed`/`failed`/`aborted` counts from an `<agent_swarm_result>`
6142/// summary line, e.g. `<summary>completed: 2, failed: 1</summary>`. Omitted
6143/// buckets default to 0. Returns `None` when the text is not a swarm result.
6144fn parse_swarm_counts(text: &str) -> Option<(usize, usize, usize)> {
6145    if !text.contains("<agent_swarm_result>") {
6146        return None;
6147    }
6148    let bucket = |label: &str| -> usize {
6149        let needle = format!("{label}: ");
6150        text.find(&needle)
6151            .map(|start| start + needle.len())
6152            .map(|start| {
6153                text[start..]
6154                    .chars()
6155                    .take_while(|c| c.is_ascii_digit())
6156                    .collect::<String>()
6157            })
6158            .and_then(|digits| digits.parse().ok())
6159            .unwrap_or(0)
6160    };
6161    Some((bucket("completed"), bucket("failed"), bucket("aborted")))
6162}
6163
6164fn effective_reasoning_for_model(cfg: &RuntimeConfig, model: &str) -> String {
6165    let base_reasoning = default_effective_reasoning_for_model(cfg, model);
6166    if cfg.dynamic_workflows.effort_profile == DynamicWorkflowEffortProfile::Ultracode {
6167        return ultracode_reasoning_level_for_model(
6168            model,
6169            &cfg.speed_policy.ultracode_reasoning,
6170            &base_reasoning,
6171        );
6172    }
6173    base_reasoning
6174}
6175
6176fn default_effective_reasoning_for_model(cfg: &RuntimeConfig, model: &str) -> String {
6177    let Some(entry) = lookup_model(model) else {
6178        return cfg
6179            .reasoning
6180            .clone()
6181            .unwrap_or_else(|| REASONING_NONE.to_string());
6182    };
6183    if entry.supported_reasoning.is_empty() {
6184        return REASONING_NONE.to_string();
6185    }
6186    cfg.reasoning
6187        .as_deref()
6188        .filter(|reasoning| {
6189            entry
6190                .supported_reasoning
6191                .iter()
6192                .any(|option| option.effort == *reasoning)
6193        })
6194        .map(str::to_string)
6195        .or_else(|| {
6196            model_profile_for_model(cfg, model)
6197                .and_then(|profile| profile.reasoning.orientation)
6198                .filter(|reasoning| {
6199                    entry
6200                        .supported_reasoning
6201                        .iter()
6202                        .any(|option| option.effort == reasoning)
6203                })
6204        })
6205        .unwrap_or_else(|| entry.default_reasoning.to_string())
6206}
6207
6208fn validate_reasoning_effort(model: &str, effort: &str) -> anyhow::Result<()> {
6209    if effort == REASONING_NONE && !model_supports_reasoning(model, effort) {
6210        return Ok(());
6211    }
6212    let Some(entry) = lookup_model(model) else {
6213        return Ok(());
6214    };
6215    if entry
6216        .supported_reasoning
6217        .iter()
6218        .any(|option| option.effort == effort)
6219    {
6220        Ok(())
6221    } else {
6222        anyhow::bail!("model {model} does not support reasoning effort {effort}")
6223    }
6224}
6225
6226fn validate_runtime_config_reasoning(cfg: &RuntimeConfig) -> anyhow::Result<()> {
6227    let Some(reasoning) = cfg.reasoning.as_deref() else {
6228        return Ok(());
6229    };
6230    let Some(entry) = lookup_model(&cfg.default_model) else {
6231        return Ok(());
6232    };
6233    if entry.provider != PROVIDER_GEMINI {
6234        return Ok(());
6235    }
6236    validate_reasoning_effort(&cfg.default_model, reasoning)
6237}
6238
6239fn validate_runtime_inference_router_config(
6240    registry: &ExtensionRegistry,
6241    cfg: &RuntimeConfig,
6242) -> anyhow::Result<()> {
6243    if !cfg.inference_router.enabled {
6244        return Ok(());
6245    }
6246    let Some(router_id) = cfg.inference_router.router_id.as_deref() else {
6247        anyhow::bail!("inference_router.enabled requires inference_router.router");
6248    };
6249    if registry.inference_router(router_id).is_some() {
6250        return Ok(());
6251    }
6252    let available = registry
6253        .inference_routers
6254        .iter()
6255        .map(|router| router.id())
6256        .collect::<Vec<_>>()
6257        .join(", ");
6258    if available.is_empty() {
6259        anyhow::bail!("inference router {router_id:?} is not registered");
6260    }
6261    anyhow::bail!(
6262        "inference router {router_id:?} is not registered; available routers: {available}"
6263    );
6264}
6265
6266fn model_supports_reasoning(model: &str, effort: &str) -> bool {
6267    lookup_model(model)
6268        .map(|entry| {
6269            entry
6270                .supported_reasoning
6271                .iter()
6272                .any(|option| option.effort == effort)
6273        })
6274        .unwrap_or(false)
6275}
6276
6277fn is_final_answer_phase(phase: Option<&str>) -> bool {
6278    phase.is_none_or(|phase| phase.is_empty() || phase == FINAL_ANSWER_PHASE)
6279}
6280
6281fn edit_tool_for_model<'a>(cfg: &'a RuntimeConfig, model: &'a str) -> Option<&'a str> {
6282    cfg.model_edit_tools
6283        .get(model)
6284        .map(String::as_str)
6285        .or_else(|| {
6286            cfg.model_profiles
6287                .get(model)
6288                .and_then(|profile| profile.edit_tool.as_deref())
6289        })
6290        .or_else(|| lookup_model(model).and_then(|entry| entry.edit_tool))
6291        .or(Some(EDIT_TOOL_EDIT))
6292}
6293
6294fn model_profile_for_model(cfg: &RuntimeConfig, model: &str) -> Option<ModelHarnessProfile> {
6295    cfg.model_profiles
6296        .get(model)
6297        .cloned()
6298        .or_else(|| built_in_model_profile(model))
6299}
6300
6301pub(crate) fn allowlist_permits(allowlist: &[String], tool_name: &str) -> bool {
6302    allowlist.is_empty() || allowlist.iter().any(|allowed| allowed == tool_name)
6303}
6304
6305/// Provider-aware profile resolution for the active turn.
6306///
6307/// Many model ids are shared across providers (for example Cursor proxies
6308/// `claude-opus-4-8`). Resolving the harness profile by id alone picks the
6309/// first catalog entry, which assigns cross-provider ids the wrong family and
6310/// instruction overlay (e.g. a `cursor/claude-opus-4-8` turn would otherwise
6311/// inherit the Anthropic overlay). Prefer the explicit `(provider, model)`
6312/// catalog entry, keeping user-configured profile overrides as the top
6313/// precedence.
6314fn model_profile_for_provider_model(
6315    cfg: &RuntimeConfig,
6316    provider: &str,
6317    model: &str,
6318) -> Option<ModelHarnessProfile> {
6319    cfg.model_profiles
6320        .get(model)
6321        .cloned()
6322        .or_else(|| built_in_model_profile_for_provider(provider, model))
6323}
6324
6325fn schema_policy_for_model(profile: Option<&ModelHarnessProfile>) -> ModelSchemaPolicy {
6326    profile
6327        .map(|profile| profile.schema_policy)
6328        .unwrap_or_default()
6329}
6330
6331fn model_profile_segment_metadata(
6332    profile: Option<&ModelHarnessProfile>,
6333    provider: &str,
6334    model: &str,
6335    segment: &str,
6336) -> serde_json::Value {
6337    serde_json::json!({
6338        "kind": MODEL_PROFILE_TRACE_KIND,
6339        "segment": segment,
6340        "provider": provider,
6341        "model": model,
6342        "profileModel": profile.map(|profile| profile.model.as_str()).unwrap_or(model),
6343        "providerFamily": profile.map(|profile| profile.provider_family),
6344        "editTool": profile.and_then(|profile| profile.edit_tool.as_deref()),
6345        "schemaPolicy": profile.map(|profile| profile.schema_policy),
6346        "instructionOverlay": profile.map(|profile| profile.instruction_overlay),
6347        "parallelToolCalls": profile.and_then(|profile| profile.parallel_tool_calls),
6348        "autoCompactTokenLimit": profile.and_then(|profile| profile.auto_compact_token_limit),
6349    })
6350}
6351
6352fn model_switch_summary(
6353    transcript: &[TranscriptItem],
6354    profile: Option<&ModelHarnessProfile>,
6355    provider: &str,
6356    model: &str,
6357    tools: &[roder_api::tools::ToolSpec],
6358) -> Option<String> {
6359    let previous = latest_model_profile_segment(transcript)?;
6360    let previous_model = previous
6361        .get("model")
6362        .and_then(serde_json::Value::as_str)
6363        .unwrap_or_default();
6364    let previous_provider = previous
6365        .get("provider")
6366        .and_then(serde_json::Value::as_str)
6367        .unwrap_or_default();
6368    if previous_model == model && previous_provider == provider {
6369        return None;
6370    }
6371
6372    let previous_profile = previous
6373        .get("profileModel")
6374        .and_then(serde_json::Value::as_str)
6375        .unwrap_or(previous_model);
6376    let current_profile = profile
6377        .map(|profile| profile.model.as_str())
6378        .unwrap_or(model);
6379    let previous_edit_tool = previous
6380        .get("editTool")
6381        .and_then(serde_json::Value::as_str)
6382        .unwrap_or("none");
6383    let current_edit_tool = profile
6384        .and_then(|profile| profile.edit_tool.as_deref())
6385        .unwrap_or("none");
6386    let tool_names = tools
6387        .iter()
6388        .map(|tool| tool.name.as_str())
6389        .take(12)
6390        .collect::<Vec<_>>()
6391        .join(", ");
6392    Some(format!(
6393        "{MODEL_SWITCH_SUMMARY_PREFIX} previous profile {previous_provider}/{previous_profile} used edit tool {previous_edit_tool}. Current profile {provider}/{current_profile} uses edit tool {current_edit_tool}. Available tools now: {}.",
6394        if tool_names.is_empty() {
6395            "none"
6396        } else {
6397            &tool_names
6398        }
6399    ))
6400}
6401
6402fn latest_model_profile_segment(transcript: &[TranscriptItem]) -> Option<&serde_json::Value> {
6403    transcript.iter().rev().find_map(|item| {
6404        let TranscriptItem::ProviderMetadata(value) = item else {
6405            return None;
6406        };
6407        (value.get("kind").and_then(serde_json::Value::as_str) == Some(MODEL_PROFILE_TRACE_KIND))
6408            .then_some(value)
6409    })
6410}
6411
6412pub fn validate_edit_tool(value: &str) -> anyhow::Result<()> {
6413    match value.trim() {
6414        EDIT_TOOL_PATCH | EDIT_TOOL_EDIT => Ok(()),
6415        _ => anyhow::bail!(
6416            "unsupported edit_tool {value:?}; allowed values: {EDIT_TOOL_PATCH}, {EDIT_TOOL_EDIT}"
6417        ),
6418    }
6419}
6420
6421fn should_persist_thread_event(thread_id: &str) -> bool {
6422    !is_synthetic_event_thread_id(thread_id)
6423}
6424
6425#[cfg(test)]
6426#[path = "runtime/codex_v2_tests.rs"]
6427mod codex_v2_tests;
6428
6429#[cfg(test)]
6430#[path = "runtime/codex_v2_lifecycle_tests.rs"]
6431mod codex_v2_lifecycle_tests;
6432
6433#[cfg(test)]
6434mod tests {
6435    use super::*;
6436    use futures::stream;
6437    use roder_api::catalog::{
6438        PROVIDER_MOCK, REASONING_HIGH, REASONING_LOW, REASONING_MEDIUM, REASONING_MINIMAL,
6439        REASONING_NONE, REASONING_XHIGH,
6440    };
6441    use roder_api::extension::ExtensionRegistryBuilder;
6442    use roder_api::inference::{
6443        CompletionMetadata, InferenceCapabilities, InferenceEngine, InferenceEventStream,
6444        InferenceProviderContext, InferenceTurnContext, MessageDelta, ModelDescriptor,
6445        ModelInstructionOverlay, ModelProfileReasoning, ModelSchemaPolicy, ProviderFamily,
6446        ReasoningEffortDescriptor,
6447    };
6448    use roder_api::inference_routing::{
6449        InferenceRouter, InferenceRoutingContext, InferenceRoutingDecision, InferenceRoutingOutcome,
6450    };
6451    use roder_api::thread::ThreadStoreFactory;
6452    use roder_api::tools::{ToolContributor, ToolExecutor, ToolSpec};
6453    use roder_ext_jsonl_thread_store::store::JsonlThreadStoreFactory;
6454    use std::sync::Mutex as StdMutex;
6455
6456    fn test_workspace() -> String {
6457        std::env::current_dir().unwrap().display().to_string()
6458    }
6459
6460    #[test]
6461    fn interactive_explicit_turn_deadline_has_bounded_remaining_time() {
6462        let config = RuntimeConfig {
6463            runtime_profile: RuntimeProfile::Interactive,
6464            turn_deadline_seconds: Some(60),
6465            ..RuntimeConfig::default()
6466        };
6467
6468        let deadline = turn_deadline_for_config(&config).expect("configured turn deadline");
6469        let remaining = deadline_remaining_seconds(Some(deadline));
6470
6471        assert!(
6472            matches!(remaining, Some(1..=60)),
6473            "interactive deadline must be positive and no longer than configured: {remaining:?}"
6474        );
6475    }
6476
6477    #[test]
6478    fn interactive_without_turn_deadline_stays_unbounded() {
6479        let config = RuntimeConfig {
6480            runtime_profile: RuntimeProfile::Interactive,
6481            turn_deadline_seconds: None,
6482            ..RuntimeConfig::default()
6483        };
6484
6485        assert_eq!(turn_deadline_for_config(&config), None);
6486    }
6487
6488    struct MetadataMissingStore;
6489
6490    #[async_trait::async_trait]
6491    impl ThreadStore for MetadataMissingStore {
6492        fn id(&self) -> roder_api::thread::ThreadStoreId {
6493            "metadata-missing-store".to_string()
6494        }
6495
6496        async fn create_thread(&self, metadata: ThreadMetadata) -> anyhow::Result<ThreadMetadata> {
6497            Ok(metadata)
6498        }
6499
6500        async fn list_threads(&self) -> anyhow::Result<Vec<ThreadMetadata>> {
6501            Ok(Vec::new())
6502        }
6503
6504        async fn load_thread(
6505            &self,
6506            _thread_id: &ThreadId,
6507        ) -> anyhow::Result<Option<ThreadSnapshot>> {
6508            Ok(Some(ThreadSnapshot {
6509                metadata: None,
6510                ..ThreadSnapshot::default()
6511            }))
6512        }
6513
6514        async fn append_event(
6515            &self,
6516            _thread_id: &ThreadId,
6517            _envelope: &EventEnvelope,
6518        ) -> anyhow::Result<()> {
6519            Ok(())
6520        }
6521    }
6522
6523    struct MetadataMissingStoreFactory;
6524
6525    impl ThreadStoreFactory for MetadataMissingStoreFactory {
6526        fn id(&self) -> roder_api::thread::ThreadStoreId {
6527            "metadata-missing-store".to_string()
6528        }
6529
6530        fn create(&self) -> Arc<dyn ThreadStore> {
6531            Arc::new(MetadataMissingStore)
6532        }
6533    }
6534
6535    #[test]
6536    fn synthetic_app_server_events_are_not_thread_events() {
6537        for thread_id in ["app-server", "runtime", "thread-workflow"] {
6538            assert!(!should_persist_thread_event(thread_id));
6539        }
6540        assert!(should_persist_thread_event("thread-discovery"));
6541        assert!(should_persist_thread_event("thread-plan"));
6542        assert!(should_persist_thread_event("thread-process"));
6543        assert!(should_persist_thread_event("thread-1"));
6544    }
6545
6546    #[test]
6547    fn server_side_compaction_uses_catalog_ninety_percent_default() {
6548        assert_eq!(
6549            server_side_compaction_threshold(&RuntimeConfig::default(), "gpt-5.5"),
6550            Some(945_000)
6551        );
6552        assert_eq!(
6553            server_side_compaction_threshold(&RuntimeConfig::default(), "gpt-5.3-codex-spark"),
6554            Some(115_200)
6555        );
6556    }
6557
6558    #[test]
6559    fn server_side_compaction_respects_explicit_config_override() {
6560        let cfg = RuntimeConfig {
6561            auto_compact_token_limit: Some(123_456),
6562            ..RuntimeConfig::default()
6563        };
6564
6565        assert_eq!(
6566            server_side_compaction_threshold(&cfg, "gpt-5.5"),
6567            Some(123_456)
6568        );
6569    }
6570
6571    #[tokio::test]
6572    async fn pre_request_compaction_runs_when_server_side_model_is_at_context_window() {
6573        let captured = Arc::new(StdMutex::new(None));
6574        let mut builder = ExtensionRegistryBuilder::new();
6575        builder.inference_engine(Arc::new(CapturingEngine {
6576            request: captured.clone(),
6577        }));
6578        let thread_root = std::env::temp_dir().join(format!(
6579            "roder-pre-request-compaction-{}",
6580            uuid::Uuid::new_v4()
6581        ));
6582        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
6583            base_path: thread_root.clone(),
6584        }));
6585        let runtime = Arc::new(
6586            Runtime::new(
6587                builder.build().unwrap(),
6588                RuntimeConfig {
6589                    default_provider: PROVIDER_MOCK.to_string(),
6590                    default_model: "gpt-5.5".to_string(),
6591                    file_backed_dynamic_context: true,
6592                    ..RuntimeConfig::default()
6593                },
6594            )
6595            .unwrap(),
6596        );
6597        let thread_id = runtime
6598            .create_thread(Some("Pre-request compaction".to_string()))
6599            .await
6600            .unwrap()
6601            .thread_id;
6602        let old_turn = "old-turn".to_string();
6603        runtime
6604            .persist_turn_item(
6605                &thread_id,
6606                &old_turn,
6607                &TranscriptItem::UserMessage(UserMessage::text("old context ".repeat(4_300_000))),
6608            )
6609            .await
6610            .unwrap();
6611
6612        let mut events = runtime.subscribe_events();
6613        runtime
6614            .start_turn(StartTurnRequest {
6615                thread_id: thread_id.clone(),
6616                message: "continue".to_string(),
6617                images: Vec::new(),
6618                provider_override: None,
6619                model_override: None,
6620                reasoning_override: None,
6621                workspace: test_workspace(),
6622                instructions: InstructionBundle::default(),
6623                developer_context: None,
6624                task_ledger_required: false,
6625                service_tier_override: None,
6626            })
6627            .await
6628            .unwrap();
6629        loop {
6630            let envelope = tokio::time::timeout(std::time::Duration::from_secs(5), events.recv())
6631                .await
6632                .unwrap()
6633                .unwrap();
6634            if envelope.thread_id.as_deref() == Some(&thread_id)
6635                && matches!(envelope.event, RoderEvent::TurnCompleted(_))
6636            {
6637                break;
6638            }
6639        }
6640
6641        let request = captured.lock().unwrap().clone().unwrap();
6642        assert!(
6643            matches!(
6644                request.transcript.first(),
6645                Some(TranscriptItem::ContextCompaction(_))
6646            ),
6647            "provider request should start with a local emergency compaction item"
6648        );
6649        assert!(
6650            request.transcript.len() < 4,
6651            "provider request should not replay the full oversized prior transcript: {:?}",
6652            request.transcript
6653        );
6654
6655        let _ = std::fs::remove_dir_all(thread_root);
6656    }
6657
6658    #[tokio::test]
6659    async fn continue_after_context_window_failure_compacts_before_provider_request() {
6660        let captured = Arc::new(StdMutex::new(None));
6661        let mut builder = ExtensionRegistryBuilder::new();
6662        builder.inference_engine(Arc::new(CapturingEngine {
6663            request: captured.clone(),
6664        }));
6665        let thread_root = std::env::temp_dir().join(format!(
6666            "roder-context-failure-continue-{}",
6667            uuid::Uuid::new_v4()
6668        ));
6669        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
6670            base_path: thread_root.clone(),
6671        }));
6672        let runtime = Arc::new(
6673            Runtime::new(
6674                builder.build().unwrap(),
6675                RuntimeConfig {
6676                    default_provider: PROVIDER_MOCK.to_string(),
6677                    default_model: "gpt-5.5".to_string(),
6678                    file_backed_dynamic_context: true,
6679                    ..RuntimeConfig::default()
6680                },
6681            )
6682            .unwrap(),
6683        );
6684        let thread_id = runtime
6685            .create_thread(Some("Context failure continue".to_string()))
6686            .await
6687            .unwrap()
6688            .thread_id;
6689        let failed_turn = "failed-turn".to_string();
6690        runtime
6691            .persist_turn_item(
6692                &thread_id,
6693                &failed_turn,
6694                &TranscriptItem::UserMessage(UserMessage::text("old work ".repeat(10_000))),
6695            )
6696            .await
6697            .unwrap();
6698        runtime
6699            .persist_turn_item(
6700                &thread_id,
6701                &failed_turn,
6702                &TranscriptItem::Error(ErrorRecord {
6703                    message: "Your input exceeds the context window of this model. Please adjust your input and try again."
6704                        .to_string(),
6705                }),
6706            )
6707            .await
6708            .unwrap();
6709
6710        let mut events = runtime.subscribe_events();
6711        runtime
6712            .start_turn(StartTurnRequest {
6713                thread_id: thread_id.clone(),
6714                message: "continue".to_string(),
6715                images: Vec::new(),
6716                provider_override: None,
6717                model_override: None,
6718                reasoning_override: None,
6719                workspace: test_workspace(),
6720                instructions: InstructionBundle::default(),
6721                developer_context: None,
6722                task_ledger_required: false,
6723                service_tier_override: None,
6724            })
6725            .await
6726            .unwrap();
6727        loop {
6728            let envelope = tokio::time::timeout(std::time::Duration::from_secs(5), events.recv())
6729                .await
6730                .unwrap()
6731                .unwrap();
6732            if envelope.thread_id.as_deref() == Some(&thread_id)
6733                && matches!(envelope.event, RoderEvent::TurnCompleted(_))
6734            {
6735                break;
6736            }
6737        }
6738
6739        let request = captured.lock().unwrap().clone().unwrap();
6740        assert!(
6741            matches!(
6742                request.transcript.first(),
6743                Some(TranscriptItem::ContextCompaction(_))
6744            ),
6745            "provider request after context-window failure should start with local compaction"
6746        );
6747        assert!(
6748            request
6749                .transcript
6750                .iter()
6751                .any(|item| matches!(item, TranscriptItem::UserMessage(message) if message.text == "continue")),
6752            "current continue prompt must be preserved: {:?}",
6753            request.transcript
6754        );
6755        assert!(
6756            !request.transcript.iter().any(
6757                |item| matches!(item, TranscriptItem::Error(error) if error.message.contains("context window"))
6758            ),
6759            "raw prior context-window error should be summarized, not replayed: {:?}",
6760            request.transcript
6761        );
6762
6763        let _ = std::fs::remove_dir_all(thread_root);
6764    }
6765
6766    #[tokio::test]
6767    async fn workspace_for_thread_falls_back_when_metadata_is_missing() {
6768        let workspace = test_workspace();
6769        let mut builder = ExtensionRegistryBuilder::new();
6770        builder.inference_engine(Arc::new(FakeInferenceEngine));
6771        builder.thread_store_factory(Arc::new(MetadataMissingStoreFactory));
6772        let runtime = Runtime::new(
6773            builder.build().unwrap(),
6774            RuntimeConfig {
6775                workspace: Some(workspace.clone()),
6776                ..RuntimeConfig::default()
6777            },
6778        )
6779        .unwrap();
6780
6781        let resolved = runtime
6782            .workspace_for_thread(&ThreadId::from("thread-workflow"))
6783            .await
6784            .unwrap();
6785
6786        assert_eq!(resolved, workspace);
6787    }
6788
6789    #[tokio::test]
6790    async fn automations_can_create_project_thread_with_model_overrides() {
6791        let runtime = Runtime::fake().unwrap();
6792        let workspace = std::env::temp_dir().join("project");
6793        let metadata = runtime
6794            .create_thread_with(CreateThreadRequest {
6795                title: Some("Automation: nightly status".to_string()),
6796                workspace: workspace.display().to_string(),
6797                workspace_id: None,
6798                root_id: None,
6799                provider: Some("mock".to_string()),
6800                model: Some("mock".to_string()),
6801                selection_mode: None,
6802                tool_allowlist: Vec::new(),
6803                developer_instructions: None,
6804                external_tools: Vec::new(),
6805                runner: None,
6806            })
6807            .await
6808            .unwrap();
6809
6810        assert_eq!(
6811            metadata.title.as_deref(),
6812            Some("Automation: nightly status")
6813        );
6814        assert_eq!(metadata.workspace, workspace.display().to_string());
6815        assert_eq!(metadata.provider.as_deref(), Some("mock"));
6816        assert_eq!(metadata.model.as_deref(), Some("mock"));
6817    }
6818
6819    #[tokio::test]
6820    async fn prior_provider_compaction_boundary_is_used_on_next_turn() {
6821        let captured = Arc::new(StdMutex::new(None));
6822        let mut builder = ExtensionRegistryBuilder::new();
6823        builder.inference_engine(Arc::new(CapturingEngine {
6824            request: captured.clone(),
6825        }));
6826        let thread_root = std::env::temp_dir().join(format!(
6827            "roder-provider-compaction-boundary-{}",
6828            uuid::Uuid::new_v4()
6829        ));
6830        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
6831            base_path: thread_root.clone(),
6832        }));
6833        let runtime = Arc::new(
6834            Runtime::new(
6835                builder.build().unwrap(),
6836                RuntimeConfig {
6837                    default_provider: PROVIDER_MOCK.to_string(),
6838                    default_model: "gpt-5.5".to_string(),
6839                    ..RuntimeConfig::default()
6840                },
6841            )
6842            .unwrap(),
6843        );
6844        let thread_id = runtime
6845            .create_thread(Some("Provider compaction boundary".to_string()))
6846            .await
6847            .unwrap()
6848            .thread_id;
6849        let old_turn = "old-turn".to_string();
6850        runtime
6851            .persist_turn_item(
6852                &thread_id,
6853                &old_turn,
6854                &TranscriptItem::UserMessage(UserMessage::text(
6855                    "old history that must not be replayed after provider compaction",
6856                )),
6857            )
6858            .await
6859            .unwrap();
6860        runtime
6861            .persist_turn_item(
6862                &thread_id,
6863                &old_turn,
6864                &TranscriptItem::AssistantMessage(AssistantMessage {
6865                    text: "old answer".to_string(),
6866                    phase: None,
6867                }),
6868            )
6869            .await
6870            .unwrap();
6871        runtime
6872            .persist_turn_item(
6873                &thread_id,
6874                &old_turn,
6875                &TranscriptItem::ProviderMetadata(serde_json::json!({
6876                    "output": [{
6877                        "id": "cmp_1",
6878                        "type": "compaction",
6879                        "encrypted_content": "opaque-state"
6880                    }]
6881                })),
6882            )
6883            .await
6884            .unwrap();
6885        runtime
6886            .persist_turn_item(
6887                &thread_id,
6888                &old_turn,
6889                &TranscriptItem::AssistantMessage(AssistantMessage {
6890                    text: "after compact".to_string(),
6891                    phase: None,
6892                }),
6893            )
6894            .await
6895            .unwrap();
6896
6897        let mut events = runtime.subscribe_events();
6898        runtime
6899            .start_turn(StartTurnRequest {
6900                thread_id: thread_id.clone(),
6901                message: "continue".to_string(),
6902                images: Vec::new(),
6903                provider_override: None,
6904                model_override: None,
6905                reasoning_override: None,
6906                workspace: test_workspace(),
6907                instructions: InstructionBundle::default(),
6908                developer_context: None,
6909                task_ledger_required: false,
6910                service_tier_override: None,
6911            })
6912            .await
6913            .unwrap();
6914        loop {
6915            let envelope = tokio::time::timeout(std::time::Duration::from_secs(5), events.recv())
6916                .await
6917                .unwrap()
6918                .unwrap();
6919            if envelope.thread_id.as_deref() == Some(&thread_id)
6920                && matches!(envelope.event, RoderEvent::TurnCompleted(_))
6921            {
6922                break;
6923            }
6924        }
6925
6926        let request = captured.lock().unwrap().clone().unwrap();
6927        let compaction_idx = request.transcript.iter().position(|item| {
6928            matches!(
6929                item,
6930                TranscriptItem::ProviderMetadata(metadata)
6931                    if crate::compaction::provider_metadata_has_compaction(metadata)
6932            )
6933        });
6934        assert!(
6935            compaction_idx.is_some(),
6936            "next provider request should include the provider compaction item: {:?}",
6937            request.transcript
6938        );
6939        // Skill/context injectors may prepend non-history items, but no prior-turn
6940        // conversation may appear before the latest provider compaction boundary.
6941        let history_before_boundary =
6942            request
6943                .transcript
6944                .iter()
6945                .take(compaction_idx.unwrap())
6946                .any(|item| match item {
6947                    TranscriptItem::AssistantMessage(_)
6948                    | TranscriptItem::ToolCall(_)
6949                    | TranscriptItem::ToolResult(_) => true,
6950                    TranscriptItem::UserMessage(message) => {
6951                        !message.text.contains("<skills>") && message.text != "continue"
6952                    }
6953                    _ => false,
6954                });
6955        assert!(
6956            !history_before_boundary,
6957            "pre-provider-compaction conversation must not be replayed: {:?}",
6958            request.transcript
6959        );
6960        assert!(
6961            !request.transcript.iter().any(|item| {
6962                matches!(
6963                    item,
6964                    TranscriptItem::UserMessage(message)
6965                        if message.text.contains("old history that must not be replayed")
6966                )
6967            }),
6968            "pre-provider-compaction history must not be replayed: {:?}",
6969            request.transcript
6970        );
6971        assert!(
6972            request.transcript.iter().any(|item| {
6973                matches!(
6974                    item,
6975                    TranscriptItem::UserMessage(message) if message.text == "continue"
6976                )
6977            }),
6978            "current continue prompt must be preserved: {:?}",
6979            request.transcript
6980        );
6981        assert_eq!(
6982            request.runtime.auto_compact_token_limit,
6983            Some(945_000),
6984            "gpt-5.5 should keep server-side auto compaction configured"
6985        );
6986
6987        let _ = std::fs::remove_dir_all(thread_root);
6988    }
6989
6990    #[test]
6991    fn server_side_compaction_is_only_enabled_for_supported_models() {
6992        let cfg = RuntimeConfig {
6993            auto_compact_token_limit: Some(123_456),
6994            ..RuntimeConfig::default()
6995        };
6996
6997        assert_eq!(server_side_compaction_threshold(&cfg, "mock"), None);
6998        assert_eq!(
6999            server_side_compaction_threshold(&cfg, "codex-auto-review"),
7000            None
7001        );
7002    }
7003
7004    #[test]
7005    fn reasoning_is_disabled_for_models_without_reasoning_support() {
7006        let cfg = RuntimeConfig {
7007            reasoning: Some(REASONING_HIGH.to_string()),
7008            ..RuntimeConfig::default()
7009        };
7010
7011        assert_eq!(
7012            effective_reasoning_for_model(&cfg, "claude-haiku-4-5-20251001"),
7013            REASONING_NONE
7014        );
7015        assert_eq!(
7016            reasoning_for_model(&cfg, "claude-haiku-4-5-20251001"),
7017            ReasoningConfig::default()
7018        );
7019    }
7020
7021    #[test]
7022    fn unsupported_configured_reasoning_falls_back_to_model_default() {
7023        let cfg = RuntimeConfig {
7024            reasoning: Some(REASONING_MINIMAL.to_string()),
7025            ..RuntimeConfig::default()
7026        };
7027
7028        assert_eq!(
7029            effective_reasoning_for_model(&cfg, "gpt-5.5"),
7030            REASONING_MEDIUM
7031        );
7032    }
7033
7034    #[test]
7035    fn unsupported_configured_gemini_reasoning_is_rejected() {
7036        let mut builder = ExtensionRegistryBuilder::new();
7037        builder.inference_engine(std::sync::Arc::new(FakeInferenceEngine));
7038
7039        let err = match Runtime::new(
7040            builder.build().unwrap(),
7041            RuntimeConfig {
7042                default_model: "gemini-3.5-flash".to_string(),
7043                reasoning: Some(REASONING_XHIGH.to_string()),
7044                ..RuntimeConfig::default()
7045            },
7046        ) {
7047            Ok(_) => panic!("expected unsupported Gemini reasoning to be rejected"),
7048            Err(err) => err,
7049        };
7050
7051        assert!(
7052            err.to_string()
7053                .contains("model gemini-3.5-flash does not support reasoning effort xhigh")
7054        );
7055    }
7056
7057    #[tokio::test]
7058    async fn selecting_none_for_non_reasoning_model_preserves_stored_preference() {
7059        let runtime = Runtime::new(
7060            Runtime::fake().unwrap().registry,
7061            RuntimeConfig {
7062                reasoning: Some(REASONING_HIGH.to_string()),
7063                ..RuntimeConfig::default()
7064            },
7065        )
7066        .unwrap();
7067
7068        let cfg = runtime
7069            .select_provider(
7070                roder_api::catalog::PROVIDER_MOCK.to_string(),
7071                Some("claude-haiku-4-5-20251001".to_string()),
7072                Some(REASONING_NONE.to_string()),
7073            )
7074            .await
7075            .unwrap();
7076
7077        assert_eq!(cfg.reasoning.as_deref(), Some(REASONING_HIGH));
7078        assert_eq!(runtime.effective_reasoning().await, REASONING_NONE);
7079    }
7080
7081    #[tokio::test]
7082    async fn selecting_none_for_model_that_supports_none_updates_preference() {
7083        let runtime = Runtime::new(
7084            Runtime::fake().unwrap().registry,
7085            RuntimeConfig {
7086                reasoning: Some(REASONING_HIGH.to_string()),
7087                ..RuntimeConfig::default()
7088            },
7089        )
7090        .unwrap();
7091
7092        let cfg = runtime
7093            .select_provider(
7094                roder_api::catalog::PROVIDER_MOCK.to_string(),
7095                Some("mock".to_string()),
7096                Some(REASONING_NONE.to_string()),
7097            )
7098            .await
7099            .unwrap();
7100
7101        assert_eq!(cfg.reasoning.as_deref(), Some(REASONING_NONE));
7102    }
7103
7104    #[test]
7105    fn parallel_tool_calls_default_on_with_model_override() {
7106        assert!(parallel_tool_calls_for_model(
7107            &RuntimeConfig::default(),
7108            "custom-model"
7109        ));
7110
7111        let cfg = RuntimeConfig {
7112            model_parallel_tool_calls: std::collections::HashMap::from([(
7113                "custom-model".to_string(),
7114                false,
7115            )]),
7116            ..RuntimeConfig::default()
7117        };
7118
7119        assert!(!parallel_tool_calls_for_model(&cfg, "custom-model"));
7120        assert!(parallel_tool_calls_for_model(&cfg, "other-model"));
7121    }
7122
7123    #[test]
7124    fn profile_parallel_tool_calls_applies_between_config_and_default() {
7125        let cfg = RuntimeConfig {
7126            model_profiles: std::collections::HashMap::from([(
7127                "gpt-5.5".to_string(),
7128                test_model_profile("gpt-5.5"),
7129            )]),
7130            ..RuntimeConfig::default()
7131        };
7132
7133        assert!(!parallel_tool_calls_for_model(&cfg, "gpt-5.5"));
7134
7135        let cfg = RuntimeConfig {
7136            model_parallel_tool_calls: std::collections::HashMap::from([(
7137                "gpt-5.5".to_string(),
7138                true,
7139            )]),
7140            ..cfg
7141        };
7142
7143        assert!(parallel_tool_calls_for_model(&cfg, "gpt-5.5"));
7144    }
7145
7146    struct CapturingEngine {
7147        request: Arc<StdMutex<Option<AgentInferenceRequest>>>,
7148    }
7149
7150    #[async_trait::async_trait]
7151    impl InferenceEngine for CapturingEngine {
7152        fn id(&self) -> String {
7153            roder_api::catalog::PROVIDER_MOCK.to_string()
7154        }
7155
7156        fn capabilities(&self) -> InferenceCapabilities {
7157            InferenceCapabilities::coding_agent_default()
7158        }
7159
7160        async fn list_models(
7161            &self,
7162            _ctx: InferenceProviderContext<'_>,
7163        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7164            Ok(roder_api::catalog::models_for_provider(
7165                roder_api::catalog::PROVIDER_MOCK,
7166                true,
7167            ))
7168        }
7169
7170        async fn stream_turn(
7171            &self,
7172            _ctx: InferenceTurnContext<'_>,
7173            request: AgentInferenceRequest,
7174        ) -> anyhow::Result<InferenceEventStream> {
7175            *self.request.lock().unwrap() = Some(request);
7176            Ok(Box::pin(stream::iter(vec![
7177                Ok(InferenceEvent::MessageDelta(MessageDelta {
7178                    text: "done".to_string(),
7179                    phase: None,
7180                })),
7181                Ok(InferenceEvent::Completed(CompletionMetadata {
7182                    stop_reason: Some("stop".to_string()),
7183                    provider_response_id: None,
7184                })),
7185            ])))
7186        }
7187    }
7188
7189    struct RoutingCaptureEngine {
7190        id: &'static str,
7191        models: Vec<ModelDescriptor>,
7192        requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
7193    }
7194
7195    #[async_trait::async_trait]
7196    impl InferenceEngine for RoutingCaptureEngine {
7197        fn id(&self) -> String {
7198            self.id.to_string()
7199        }
7200
7201        fn capabilities(&self) -> InferenceCapabilities {
7202            InferenceCapabilities::coding_agent_default()
7203        }
7204
7205        async fn list_models(
7206            &self,
7207            _ctx: InferenceProviderContext<'_>,
7208        ) -> anyhow::Result<Vec<ModelDescriptor>> {
7209            Ok(self.models.clone())
7210        }
7211
7212        async fn stream_turn(
7213            &self,
7214            _ctx: InferenceTurnContext<'_>,
7215            request: AgentInferenceRequest,
7216        ) -> anyhow::Result<InferenceEventStream> {
7217            self.requests.lock().unwrap().push(request);
7218            Ok(Box::pin(stream::iter(vec![
7219                Ok(InferenceEvent::MessageDelta(MessageDelta {
7220                    text: "routed".to_string(),
7221                    phase: None,
7222                })),
7223                Ok(InferenceEvent::Completed(CompletionMetadata {
7224                    stop_reason: Some("stop".to_string()),
7225                    provider_response_id: None,
7226                })),
7227            ])))
7228        }
7229    }
7230
7231    struct StaticRouter {
7232        id: &'static str,
7233        decision: InferenceRoutingDecision,
7234        contexts: Arc<StdMutex<Vec<InferenceRoutingContext>>>,
7235    }
7236
7237    #[async_trait::async_trait]
7238    impl InferenceRouter for StaticRouter {
7239        fn id(&self) -> String {
7240            self.id.to_string()
7241        }
7242
7243        async fn route(
7244            &self,
7245            context: InferenceRoutingContext,
7246        ) -> anyhow::Result<InferenceRoutingDecision> {
7247            self.contexts.lock().unwrap().push(context);
7248            Ok(self.decision.clone())
7249        }
7250    }
7251
7252    fn routing_test_model(id: &str, supported_reasoning: &[&str]) -> ModelDescriptor {
7253        ModelDescriptor {
7254            id: id.to_string(),
7255            name: id.to_string(),
7256            context_window: Some(128_000),
7257            default_reasoning: supported_reasoning
7258                .first()
7259                .map(|effort| (*effort).to_string()),
7260            supported_reasoning: supported_reasoning
7261                .iter()
7262                .map(|effort| ReasoningEffortDescriptor {
7263                    effort: (*effort).to_string(),
7264                    description: format!("{effort} reasoning"),
7265                })
7266                .collect(),
7267        }
7268    }
7269
7270    struct TaskLedgerCompletionGateEngine {
7271        calls: StdMutex<u32>,
7272        requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
7273    }
7274
7275    #[async_trait::async_trait]
7276    impl InferenceEngine for TaskLedgerCompletionGateEngine {
7277        fn id(&self) -> String {
7278            roder_api::catalog::PROVIDER_MOCK.to_string()
7279        }
7280
7281        fn capabilities(&self) -> InferenceCapabilities {
7282            InferenceCapabilities::coding_agent_default()
7283        }
7284
7285        async fn list_models(
7286            &self,
7287            _ctx: InferenceProviderContext<'_>,
7288        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7289            Ok(roder_api::catalog::models_for_provider(
7290                roder_api::catalog::PROVIDER_MOCK,
7291                true,
7292            ))
7293        }
7294
7295        async fn stream_turn(
7296            &self,
7297            _ctx: InferenceTurnContext<'_>,
7298            request: AgentInferenceRequest,
7299        ) -> anyhow::Result<InferenceEventStream> {
7300            self.requests.lock().unwrap().push(request);
7301            let mut calls = self.calls.lock().unwrap();
7302            *calls += 1;
7303            let events = match *calls {
7304                1 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7305                    id: "ledger-open".to_string(),
7306                    name: TASK_LEDGER_TOOL_NAME.to_string(),
7307                    arguments: serde_json::json!({
7308                        "tasks": [
7309                            {
7310                                "id": "inspect",
7311                                "content": "Inspect local assets",
7312                                "status": "completed",
7313                                "evidence": "listed workspace"
7314                            },
7315                            {
7316                                "id": "write",
7317                                "content": "Write /app/result.txt",
7318                                "status": "pending"
7319                            }
7320                        ],
7321                        "requireCompletionEvidence": true
7322                    })
7323                    .to_string(),
7324                }))],
7325                3 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7326                    id: "ledger-complete".to_string(),
7327                    name: TASK_LEDGER_TOOL_NAME.to_string(),
7328                    arguments: serde_json::json!({
7329                        "tasks": [
7330                            {
7331                                "id": "inspect",
7332                                "content": "Inspect local assets",
7333                                "status": "completed",
7334                                "evidence": "listed workspace"
7335                            },
7336                            {
7337                                "id": "write",
7338                                "content": "Write /app/result.txt",
7339                                "status": "completed",
7340                                "evidence": "wrote answer"
7341                            }
7342                        ],
7343                        "requireCompletionEvidence": true
7344                    })
7345                    .to_string(),
7346                }))],
7347                _ => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
7348                    text: "final".to_string(),
7349                    phase: None,
7350                }))],
7351            };
7352            Ok(Box::pin(stream::iter(events.into_iter().chain(
7353                std::iter::once(Ok(InferenceEvent::Completed(CompletionMetadata {
7354                    stop_reason: Some("stop".to_string()),
7355                    provider_response_id: None,
7356                }))),
7357            ))))
7358        }
7359    }
7360
7361    struct VerificationGateEngine {
7362        calls: StdMutex<u32>,
7363    }
7364
7365    #[async_trait::async_trait]
7366    impl InferenceEngine for VerificationGateEngine {
7367        fn id(&self) -> String {
7368            roder_api::catalog::PROVIDER_MOCK.to_string()
7369        }
7370
7371        fn capabilities(&self) -> InferenceCapabilities {
7372            InferenceCapabilities::coding_agent_default()
7373        }
7374
7375        async fn list_models(
7376            &self,
7377            _ctx: InferenceProviderContext<'_>,
7378        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7379            Ok(roder_api::catalog::models_for_provider(
7380                roder_api::catalog::PROVIDER_MOCK,
7381                true,
7382            ))
7383        }
7384
7385        async fn stream_turn(
7386            &self,
7387            _ctx: InferenceTurnContext<'_>,
7388            request: AgentInferenceRequest,
7389        ) -> anyhow::Result<InferenceEventStream> {
7390            let mut calls = self.calls.lock().unwrap();
7391            *calls += 1;
7392            let events = match *calls {
7393                1 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7394                    id: "write-1".to_string(),
7395                    name: "write_file".to_string(),
7396                    arguments: serde_json::json!({
7397                        "path": "src/lib.rs",
7398                        "content": "pub fn answer() -> u8 { 42 }\n"
7399                    })
7400                    .to_string(),
7401                }))],
7402                2 => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
7403                    text: "done too early".to_string(),
7404                    phase: None,
7405                }))],
7406                3 if request.transcript.iter().any(|item| {
7407                    matches!(
7408                        item,
7409                        TranscriptItem::UserMessage(message)
7410                            if message.text.contains("Verification gate blocked final completion")
7411                    )
7412                }) =>
7413                {
7414                    vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7415                        id: "verify-1".to_string(),
7416                        name: crate::verification_gate::VERIFICATION_TOOL_NAME.to_string(),
7417                        arguments: serde_json::json!({
7418                            "originalTask": "write code",
7419                            "changedFiles": ["src/lib.rs"],
7420                            "toolEvidence": ["write_file wrote src/lib.rs"],
7421                            "testsRun": ["cargo test -p roder-core verification_gate"],
7422                            "openGaps": [],
7423                            "status": "completed"
7424                        })
7425                        .to_string(),
7426                    }))]
7427                }
7428                _ => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
7429                    text: "verified final".to_string(),
7430                    phase: None,
7431                }))],
7432            };
7433            Ok(Box::pin(stream::iter(events.into_iter().chain(
7434                std::iter::once(Ok(InferenceEvent::Completed(CompletionMetadata {
7435                    stop_reason: Some("stop".to_string()),
7436                    provider_response_id: None,
7437                }))),
7438            ))))
7439        }
7440    }
7441
7442    struct SpeedPolicyEngine {
7443        calls: StdMutex<u32>,
7444        requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
7445    }
7446
7447    #[async_trait::async_trait]
7448    impl InferenceEngine for SpeedPolicyEngine {
7449        fn id(&self) -> String {
7450            roder_api::catalog::PROVIDER_MOCK.to_string()
7451        }
7452
7453        fn capabilities(&self) -> InferenceCapabilities {
7454            InferenceCapabilities::coding_agent_default()
7455        }
7456
7457        async fn list_models(
7458            &self,
7459            _ctx: InferenceProviderContext<'_>,
7460        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7461            Ok(roder_api::catalog::models_for_provider(
7462                roder_api::catalog::PROVIDER_MOCK,
7463                true,
7464            ))
7465        }
7466
7467        async fn stream_turn(
7468            &self,
7469            _ctx: InferenceTurnContext<'_>,
7470            request: AgentInferenceRequest,
7471        ) -> anyhow::Result<InferenceEventStream> {
7472            self.requests.lock().unwrap().push(request.clone());
7473            let mut calls = self.calls.lock().unwrap();
7474            *calls += 1;
7475            let events = match *calls {
7476                1 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7477                    id: "write-1".to_string(),
7478                    name: "write_file".to_string(),
7479                    arguments: serde_json::json!({
7480                        "path": "src/lib.rs",
7481                        "content": "pub fn answer() -> u8 { 42 }\n"
7482                    })
7483                    .to_string(),
7484                }))],
7485                3 if request.transcript.iter().any(|item| {
7486                    matches!(
7487                        item,
7488                        TranscriptItem::UserMessage(message)
7489                            if message.text.contains("Verification gate blocked final completion")
7490                    )
7491                }) =>
7492                {
7493                    vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7494                        id: "verify-1".to_string(),
7495                        name: crate::verification_gate::VERIFICATION_TOOL_NAME.to_string(),
7496                        arguments: serde_json::json!({
7497                            "originalTask": "write code",
7498                            "changedFiles": ["src/lib.rs"],
7499                            "toolEvidence": ["write_file wrote src/lib.rs"],
7500                            "testsRun": ["cargo test -p roder-core speed_policy"],
7501                            "openGaps": [],
7502                            "status": "completed"
7503                        })
7504                        .to_string(),
7505                    }))]
7506                }
7507                _ => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
7508                    text: "done".to_string(),
7509                    phase: None,
7510                }))],
7511            };
7512            Ok(Box::pin(stream::iter(events.into_iter().chain(
7513                std::iter::once(Ok(InferenceEvent::Completed(CompletionMetadata {
7514                    stop_reason: Some("stop".to_string()),
7515                    provider_response_id: None,
7516                }))),
7517            ))))
7518        }
7519    }
7520
7521    struct SwitchCaptureEngine {
7522        requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
7523    }
7524
7525    #[async_trait::async_trait]
7526    impl InferenceEngine for SwitchCaptureEngine {
7527        fn id(&self) -> String {
7528            roder_api::catalog::PROVIDER_MOCK.to_string()
7529        }
7530
7531        fn capabilities(&self) -> InferenceCapabilities {
7532            InferenceCapabilities::coding_agent_default()
7533        }
7534
7535        async fn list_models(
7536            &self,
7537            _ctx: InferenceProviderContext<'_>,
7538        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7539            Ok(roder_api::catalog::models_for_provider(
7540                roder_api::catalog::PROVIDER_MOCK,
7541                true,
7542            ))
7543        }
7544
7545        async fn stream_turn(
7546            &self,
7547            _ctx: InferenceTurnContext<'_>,
7548            request: AgentInferenceRequest,
7549        ) -> anyhow::Result<InferenceEventStream> {
7550            self.requests.lock().unwrap().push(request);
7551            Ok(Box::pin(stream::iter(vec![
7552                Ok(InferenceEvent::MessageDelta(MessageDelta {
7553                    text: "done".to_string(),
7554                    phase: None,
7555                })),
7556                Ok(InferenceEvent::Completed(CompletionMetadata {
7557                    stop_reason: Some("stop".to_string()),
7558                    provider_response_id: None,
7559                })),
7560            ])))
7561        }
7562    }
7563
7564    struct DeadlineEngine;
7565
7566    #[async_trait::async_trait]
7567    impl InferenceEngine for DeadlineEngine {
7568        fn id(&self) -> String {
7569            roder_api::catalog::PROVIDER_MOCK.to_string()
7570        }
7571
7572        fn capabilities(&self) -> InferenceCapabilities {
7573            InferenceCapabilities::coding_agent_default()
7574        }
7575
7576        async fn list_models(
7577            &self,
7578            _ctx: InferenceProviderContext<'_>,
7579        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7580            Ok(Vec::new())
7581        }
7582
7583        async fn stream_turn(
7584            &self,
7585            _ctx: InferenceTurnContext<'_>,
7586            _request: AgentInferenceRequest,
7587        ) -> anyhow::Result<InferenceEventStream> {
7588            Ok(Box::pin(stream::once(async {
7589                tokio::time::sleep(std::time::Duration::from_secs(60)).await;
7590                Ok(InferenceEvent::MessageDelta(MessageDelta {
7591                    text: "too late".to_string(),
7592                    phase: None,
7593                }))
7594            })))
7595        }
7596    }
7597
7598    struct WriteFileContributor;
7599
7600    impl ToolContributor for WriteFileContributor {
7601        fn id(&self) -> String {
7602            "test-write".to_string()
7603        }
7604
7605        fn contribute(&self, registry: &mut ToolRegistry) -> anyhow::Result<()> {
7606            registry.register(Arc::new(WriteFileTool))
7607        }
7608    }
7609
7610    struct WriteFileTool;
7611
7612    #[async_trait::async_trait]
7613    impl ToolExecutor for WriteFileTool {
7614        fn spec(&self) -> ToolSpec {
7615            ToolSpec {
7616                name: "write_file".to_string(),
7617                description: "Write a test file.".to_string(),
7618                parameters: serde_json::json!({
7619                    "type": "object",
7620                    "properties": {
7621                        "path": { "type": "string" },
7622                        "content": { "type": "string" }
7623                    },
7624                    "required": ["path", "content"],
7625                    "additionalProperties": false
7626                }),
7627            }
7628        }
7629
7630        async fn execute(
7631            &self,
7632            _ctx: ToolExecutionContext,
7633            call: ToolCall,
7634        ) -> anyhow::Result<ToolResult> {
7635            let path = call
7636                .arguments
7637                .get("path")
7638                .and_then(serde_json::Value::as_str)
7639                .unwrap_or("src/lib.rs");
7640            Ok(ToolResult {
7641                id: call.id,
7642                name: call.name,
7643                text: format!("wrote {path}"),
7644                data: serde_json::json!({ "path": path }),
7645                is_error: false,
7646            })
7647        }
7648    }
7649
7650    struct ProfileToolContributor;
7651
7652    impl ToolContributor for ProfileToolContributor {
7653        fn id(&self) -> String {
7654            "profile-tools".to_string()
7655        }
7656
7657        fn contribute(&self, registry: &mut ToolRegistry) -> anyhow::Result<()> {
7658            for name in ["apply_patch", "edit", "multi_edit", "write_file"] {
7659                registry.register(Arc::new(ProfileTool {
7660                    name: name.to_string(),
7661                }))?;
7662            }
7663            Ok(())
7664        }
7665    }
7666
7667    struct ProfileTool {
7668        name: String,
7669    }
7670
7671    #[async_trait::async_trait]
7672    impl ToolExecutor for ProfileTool {
7673        fn spec(&self) -> ToolSpec {
7674            ToolSpec {
7675                name: self.name.clone(),
7676                description: format!("{} test tool", self.name),
7677                parameters: serde_json::json!({
7678                    "type": "object",
7679                    "properties": {
7680                        "path": { "type": "string" },
7681                        "content": { "type": "string" }
7682                    },
7683                    "required": ["path", "content"],
7684                    "additionalProperties": false
7685                }),
7686            }
7687        }
7688
7689        async fn execute(
7690            &self,
7691            _ctx: ToolExecutionContext,
7692            call: ToolCall,
7693        ) -> anyhow::Result<ToolResult> {
7694            Ok(ToolResult {
7695                id: call.id,
7696                name: call.name,
7697                text: "ok".to_string(),
7698                data: serde_json::json!({}),
7699                is_error: false,
7700            })
7701        }
7702    }
7703
7704    fn test_model_profile(model: &str) -> ModelHarnessProfile {
7705        ModelHarnessProfile {
7706            model: model.to_string(),
7707            provider: roder_api::catalog::PROVIDER_OPENAI.to_string(),
7708            provider_family: ProviderFamily::OpenAi,
7709            edit_tool: Some(EDIT_TOOL_EDIT.to_string()),
7710            schema_policy: ModelSchemaPolicy::StandardRequiredFirst,
7711            instruction_overlay: ModelInstructionOverlay::IntuitiveContext,
7712            reasoning: ModelProfileReasoning {
7713                orientation: Some(REASONING_LOW.to_string()),
7714                execution: Some(REASONING_LOW.to_string()),
7715                verification: Some(REASONING_LOW.to_string()),
7716                recovery: Some(REASONING_LOW.to_string()),
7717            },
7718            parallel_tool_calls: Some(false),
7719            auto_compact_token_limit: Some(123_000),
7720        }
7721    }
7722
7723    async fn captured_profile_request(cfg: RuntimeConfig) -> AgentInferenceRequest {
7724        let captured = Arc::new(StdMutex::new(None));
7725        let mut builder = ExtensionRegistryBuilder::new();
7726        builder.inference_engine(Arc::new(CapturingEngine {
7727            request: captured.clone(),
7728        }));
7729        builder.tool_contributor(Arc::new(ProfileToolContributor));
7730        let runtime = Arc::new(Runtime::new(builder.build().unwrap(), cfg).unwrap());
7731        let mut rx = runtime.subscribe_events();
7732        let turn_id = runtime
7733            .start_turn(StartTurnRequest {
7734                thread_id: "thread-model-profile".to_string(),
7735                message: "use profile knobs".to_string(),
7736                images: Vec::new(),
7737                provider_override: None,
7738                model_override: None,
7739                reasoning_override: None,
7740                workspace: test_workspace(),
7741                instructions: InstructionBundle {
7742                    system: None,
7743                    developer: Some("base developer".to_string()),
7744                    developer_context: None,
7745                },
7746                developer_context: None,
7747                task_ledger_required: false,
7748                service_tier_override: None,
7749            })
7750            .await
7751            .unwrap();
7752
7753        tokio::time::timeout(std::time::Duration::from_secs(5), async {
7754            loop {
7755                let envelope = rx.recv().await.unwrap();
7756                if envelope.turn_id.as_deref() != Some(&turn_id) {
7757                    continue;
7758                }
7759                match envelope.event {
7760                    RoderEvent::TurnCompleted(_) => break,
7761                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7762                    _ => {}
7763                }
7764            }
7765        })
7766        .await
7767        .unwrap();
7768
7769        captured.lock().unwrap().clone().unwrap()
7770    }
7771
7772    struct ToolThenStopEngine {
7773        calls: StdMutex<u32>,
7774    }
7775
7776    #[async_trait::async_trait]
7777    impl InferenceEngine for ToolThenStopEngine {
7778        fn id(&self) -> String {
7779            roder_api::catalog::PROVIDER_MOCK.to_string()
7780        }
7781
7782        fn capabilities(&self) -> InferenceCapabilities {
7783            InferenceCapabilities::coding_agent_default()
7784        }
7785
7786        async fn list_models(
7787            &self,
7788            _ctx: InferenceProviderContext<'_>,
7789        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7790            Ok(roder_api::catalog::models_for_provider(
7791                roder_api::catalog::PROVIDER_MOCK,
7792                true,
7793            ))
7794        }
7795
7796        async fn stream_turn(
7797            &self,
7798            _ctx: InferenceTurnContext<'_>,
7799            _request: AgentInferenceRequest,
7800        ) -> anyhow::Result<InferenceEventStream> {
7801            let mut calls = self.calls.lock().unwrap();
7802            *calls += 1;
7803            let events = match *calls {
7804                1 => vec![
7805                    Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7806                        id: "write-1".to_string(),
7807                        name: "write_file".to_string(),
7808                        arguments: serde_json::json!({
7809                            "path": "src/lib.rs",
7810                            "content": "pub fn answer() -> u8 { 42 }\n"
7811                        })
7812                        .to_string(),
7813                    })),
7814                    Ok(InferenceEvent::Completed(CompletionMetadata {
7815                        stop_reason: Some("tool_use".to_string()),
7816                        provider_response_id: None,
7817                    })),
7818                ],
7819                _ => vec![
7820                    Ok(InferenceEvent::MessageDelta(MessageDelta {
7821                        text: "final".to_string(),
7822                        phase: None,
7823                    })),
7824                    Ok(InferenceEvent::Completed(CompletionMetadata {
7825                        stop_reason: Some("end_turn".to_string()),
7826                        provider_response_id: None,
7827                    })),
7828                ],
7829            };
7830            Ok(Box::pin(stream::iter(events)))
7831        }
7832    }
7833
7834    #[tokio::test]
7835    async fn turn_completed_reports_terminal_step_finish_reason() {
7836        let mut builder = ExtensionRegistryBuilder::new();
7837        builder.inference_engine(Arc::new(ToolThenStopEngine {
7838            calls: StdMutex::new(0),
7839        }));
7840        builder.tool_contributor(Arc::new(WriteFileContributor));
7841        let runtime = Arc::new(
7842            Runtime::new(
7843                builder.build().unwrap(),
7844                RuntimeConfig {
7845                    policy_mode: PolicyMode::Bypass,
7846                    agent_swarm_mode: false,
7847                    ultra_mode: false,
7848                    ..RuntimeConfig::default()
7849                },
7850            )
7851            .unwrap(),
7852        );
7853        let mut rx = runtime.subscribe_events();
7854        let turn_id = runtime
7855            .start_turn(StartTurnRequest {
7856                thread_id: "thread-finish-reason".to_string(),
7857                message: "write then finish".to_string(),
7858                images: Vec::new(),
7859                provider_override: None,
7860                model_override: None,
7861                reasoning_override: None,
7862                workspace: test_workspace(),
7863                instructions: InstructionBundle {
7864                    system: None,
7865                    developer: None,
7866                    developer_context: None,
7867                },
7868                developer_context: None,
7869                task_ledger_required: false,
7870                service_tier_override: None,
7871            })
7872            .await
7873            .unwrap();
7874
7875        let completed = tokio::time::timeout(std::time::Duration::from_secs(5), async {
7876            loop {
7877                let envelope = rx.recv().await.unwrap();
7878                if envelope.turn_id.as_deref() != Some(&turn_id) {
7879                    continue;
7880                }
7881                match envelope.event {
7882                    RoderEvent::TurnCompleted(event) => break event,
7883                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7884                    _ => {}
7885                }
7886            }
7887        })
7888        .await
7889        .unwrap();
7890
7891        // The mid-turn tool_use stop reason must not leak; the terminal
7892        // end_turn step decides the turn's finish reason.
7893        assert_eq!(completed.finish_reason.as_deref(), Some("stop"));
7894    }
7895
7896    #[tokio::test]
7897    async fn inference_router_selection_changes_request_model_and_records_event() {
7898        let default_requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7899        let routed_requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7900        let contexts = Arc::new(StdMutex::new(Vec::<InferenceRoutingContext>::new()));
7901        let selected = ModelSelection {
7902            provider: "routed-provider".to_string(),
7903            model: "routed-model".to_string(),
7904        };
7905        let default = ModelSelection {
7906            provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7907            model: "mock".to_string(),
7908        };
7909        let decision = InferenceRoutingDecision {
7910            reasoning: Some(ReasoningConfig {
7911                enabled: true,
7912                level: Some(REASONING_LOW.to_string()),
7913            }),
7914            confidence: Some(0.91),
7915            baseline: Some(default.clone()),
7916            matched_signals: vec![roder_api::inference_routing::InferenceRoutingSignal::new(
7917                "intent", "routine",
7918            )],
7919            ..InferenceRoutingDecision::selected("test-router", selected.clone(), "routine request")
7920        };
7921
7922        let mut builder = ExtensionRegistryBuilder::new();
7923        builder.inference_engine(Arc::new(RoutingCaptureEngine {
7924            id: roder_api::catalog::PROVIDER_MOCK,
7925            models: vec![routing_test_model("mock", &[REASONING_LOW])],
7926            requests: default_requests.clone(),
7927        }));
7928        builder.inference_engine(Arc::new(RoutingCaptureEngine {
7929            id: "routed-provider",
7930            models: vec![routing_test_model(
7931                "routed-model",
7932                &[REASONING_LOW, REASONING_MEDIUM],
7933            )],
7934            requests: routed_requests.clone(),
7935        }));
7936        builder.inference_router(Arc::new(StaticRouter {
7937            id: "test-router",
7938            decision,
7939            contexts: contexts.clone(),
7940        }));
7941        let thread_root =
7942            std::env::temp_dir().join(format!("roder-routing-auto-{}", uuid::Uuid::new_v4()));
7943        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
7944            base_path: thread_root.clone(),
7945        }));
7946        let runtime = Arc::new(
7947            Runtime::new(
7948                builder.build().unwrap(),
7949                RuntimeConfig {
7950                    default_provider: default.provider.clone(),
7951                    default_model: default.model.clone(),
7952                    ..RuntimeConfig::default()
7953                },
7954            )
7955            .unwrap(),
7956        );
7957        let thread_id = runtime
7958            .create_thread_with(CreateThreadRequest {
7959                title: Some("Routing auto".to_string()),
7960                workspace: test_workspace(),
7961                workspace_id: None,
7962                root_id: None,
7963                provider: Some(default.provider.clone()),
7964                model: Some(default.model.clone()),
7965                tool_allowlist: Vec::new(),
7966                developer_instructions: None,
7967                external_tools: Vec::new(),
7968                selection_mode: Some(ModelSelectionMode::auto(
7969                    "test-router:coding",
7970                    "test-router",
7971                    "Auto: Coding",
7972                    default.clone(),
7973                    Some("coding".to_string()),
7974                    None,
7975                )),
7976                runner: None,
7977            })
7978            .await
7979            .unwrap()
7980            .thread_id;
7981        let mut rx = runtime.subscribe_events();
7982        let turn_id = runtime
7983            .start_turn(StartTurnRequest {
7984                thread_id: thread_id.clone(),
7985                message: "small cleanup".to_string(),
7986                images: Vec::new(),
7987                provider_override: None,
7988                model_override: None,
7989                reasoning_override: None,
7990                workspace: test_workspace(),
7991                instructions: InstructionBundle::default(),
7992                developer_context: None,
7993                task_ledger_required: false,
7994                service_tier_override: None,
7995            })
7996            .await
7997            .unwrap();
7998
7999        let mut routing_event = None;
8000        let mut inference_started = None;
8001        tokio::time::timeout(std::time::Duration::from_secs(5), async {
8002            loop {
8003                let envelope = rx.recv().await.unwrap();
8004                if envelope.turn_id.as_deref() != Some(&turn_id) {
8005                    continue;
8006                }
8007                match envelope.event {
8008                    RoderEvent::InferenceRoutingDecision(event) => {
8009                        routing_event = Some(event);
8010                    }
8011                    RoderEvent::InferenceStarted(event) => {
8012                        inference_started = Some(event);
8013                    }
8014                    RoderEvent::TurnCompleted(_) => break,
8015                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8016                    _ => {}
8017                }
8018            }
8019        })
8020        .await
8021        .unwrap();
8022
8023        assert!(default_requests.lock().unwrap().is_empty());
8024        let routed_requests = routed_requests.lock().unwrap();
8025        assert_eq!(routed_requests.len(), 1);
8026        assert_eq!(routed_requests[0].model, selected);
8027        assert_eq!(
8028            routed_requests[0].reasoning.level.as_deref(),
8029            Some(REASONING_LOW)
8030        );
8031        assert_eq!(
8032            routed_requests[0].metadata["inferenceRouting"]["outcome"],
8033            "selected"
8034        );
8035
8036        let routing_event = routing_event.expect("routing decision event");
8037        assert_eq!(routing_event.default_selection, default);
8038        assert_eq!(routing_event.selected_selection, selected);
8039        assert_eq!(
8040            routing_event.decision.outcome,
8041            InferenceRoutingOutcome::Selected
8042        );
8043        assert_eq!(
8044            inference_started.expect("inference started event").model,
8045            selected
8046        );
8047
8048        let contexts = contexts.lock().unwrap();
8049        assert_eq!(contexts.len(), 1);
8050        assert_eq!(contexts[0].default_selection, default);
8051        assert_eq!(contexts[0].candidates.len(), 2);
8052        assert!(
8053            contexts[0]
8054                .signals
8055                .iter()
8056                .any(|signal| signal.key == "profile" && signal.value == "coding")
8057        );
8058        let _ = std::fs::remove_dir_all(thread_root);
8059    }
8060
8061    #[tokio::test]
8062    async fn inference_router_is_bypassed_for_explicit_selection() {
8063        let requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
8064        let contexts = Arc::new(StdMutex::new(Vec::<InferenceRoutingContext>::new()));
8065        let selected = ModelSelection {
8066            provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8067            model: "mock".to_string(),
8068        };
8069
8070        let mut builder = ExtensionRegistryBuilder::new();
8071        builder.inference_engine(Arc::new(RoutingCaptureEngine {
8072            id: roder_api::catalog::PROVIDER_MOCK,
8073            models: vec![routing_test_model("mock", &[REASONING_LOW])],
8074            requests: requests.clone(),
8075        }));
8076        builder.inference_router(Arc::new(StaticRouter {
8077            id: "test-router",
8078            decision: InferenceRoutingDecision::selected(
8079                "test-router",
8080                ModelSelection {
8081                    provider: "missing".to_string(),
8082                    model: "missing".to_string(),
8083                },
8084                "would route if called",
8085            ),
8086            contexts: contexts.clone(),
8087        }));
8088        let thread_root =
8089            std::env::temp_dir().join(format!("roder-routing-explicit-{}", uuid::Uuid::new_v4()));
8090        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8091            base_path: thread_root.clone(),
8092        }));
8093        let runtime = Arc::new(
8094            Runtime::new(
8095                builder.build().unwrap(),
8096                RuntimeConfig {
8097                    default_provider: selected.provider.clone(),
8098                    default_model: selected.model.clone(),
8099                    inference_router: RuntimeInferenceRouterConfig {
8100                        enabled: true,
8101                        router_id: Some("test-router".to_string()),
8102                    },
8103                    ..RuntimeConfig::default()
8104                },
8105            )
8106            .unwrap(),
8107        );
8108        let thread_id = runtime
8109            .create_thread_with(CreateThreadRequest {
8110                title: Some("Routing explicit".to_string()),
8111                workspace: test_workspace(),
8112                workspace_id: None,
8113                root_id: None,
8114                provider: Some(selected.provider.clone()),
8115                model: Some(selected.model.clone()),
8116                tool_allowlist: Vec::new(),
8117                developer_instructions: None,
8118                external_tools: Vec::new(),
8119                selection_mode: Some(ModelSelectionMode::auto(
8120                    "test-router:default",
8121                    "test-router",
8122                    "Auto",
8123                    selected.clone(),
8124                    None,
8125                    None,
8126                )),
8127                runner: None,
8128            })
8129            .await
8130            .unwrap()
8131            .thread_id;
8132        let mut rx = runtime.subscribe_events();
8133        let turn_id = runtime
8134            .start_turn(StartTurnRequest {
8135                thread_id,
8136                message: "use explicit selection".to_string(),
8137                images: Vec::new(),
8138                provider_override: Some(selected.provider.clone()),
8139                model_override: Some(selected.model.clone()),
8140                reasoning_override: None,
8141                workspace: test_workspace(),
8142                instructions: InstructionBundle::default(),
8143                developer_context: None,
8144                task_ledger_required: false,
8145                service_tier_override: None,
8146            })
8147            .await
8148            .unwrap();
8149
8150        let mut saw_routing_event = false;
8151        tokio::time::timeout(std::time::Duration::from_secs(5), async {
8152            loop {
8153                let envelope = rx.recv().await.unwrap();
8154                if envelope.turn_id.as_deref() != Some(&turn_id) {
8155                    continue;
8156                }
8157                match envelope.event {
8158                    RoderEvent::InferenceRoutingDecision(_) => {
8159                        saw_routing_event = true;
8160                    }
8161                    RoderEvent::TurnCompleted(_) => break,
8162                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8163                    _ => {}
8164                }
8165            }
8166        })
8167        .await
8168        .unwrap();
8169
8170        assert!(!saw_routing_event);
8171        assert!(contexts.lock().unwrap().is_empty());
8172        let requests = requests.lock().unwrap();
8173        assert_eq!(requests.len(), 1);
8174        assert_eq!(requests[0].model, selected);
8175        let _ = std::fs::remove_dir_all(thread_root);
8176    }
8177
8178    #[tokio::test]
8179    async fn inference_router_is_bypassed_for_manual_selection_mode() {
8180        let requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
8181        let contexts = Arc::new(StdMutex::new(Vec::<InferenceRoutingContext>::new()));
8182        let selected = ModelSelection {
8183            provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8184            model: "mock".to_string(),
8185        };
8186
8187        let mut builder = ExtensionRegistryBuilder::new();
8188        builder.inference_engine(Arc::new(RoutingCaptureEngine {
8189            id: roder_api::catalog::PROVIDER_MOCK,
8190            models: vec![routing_test_model("mock", &[REASONING_LOW])],
8191            requests: requests.clone(),
8192        }));
8193        builder.inference_router(Arc::new(StaticRouter {
8194            id: "test-router",
8195            decision: InferenceRoutingDecision::selected(
8196                "test-router",
8197                ModelSelection {
8198                    provider: "missing".to_string(),
8199                    model: "missing".to_string(),
8200                },
8201                "would route if called",
8202            ),
8203            contexts: contexts.clone(),
8204        }));
8205        let thread_root =
8206            std::env::temp_dir().join(format!("roder-routing-manual-{}", uuid::Uuid::new_v4()));
8207        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8208            base_path: thread_root.clone(),
8209        }));
8210        let runtime = Arc::new(
8211            Runtime::new(
8212                builder.build().unwrap(),
8213                RuntimeConfig {
8214                    default_provider: selected.provider.clone(),
8215                    default_model: selected.model.clone(),
8216                    inference_router: RuntimeInferenceRouterConfig {
8217                        enabled: true,
8218                        router_id: Some("test-router".to_string()),
8219                    },
8220                    ..RuntimeConfig::default()
8221                },
8222            )
8223            .unwrap(),
8224        );
8225        let thread_id = runtime
8226            .create_thread_with(CreateThreadRequest {
8227                title: Some("Routing manual".to_string()),
8228                workspace: test_workspace(),
8229                workspace_id: None,
8230                root_id: None,
8231                provider: Some(selected.provider.clone()),
8232                model: Some(selected.model.clone()),
8233                tool_allowlist: Vec::new(),
8234                developer_instructions: None,
8235                external_tools: Vec::new(),
8236                selection_mode: Some(ModelSelectionMode::manual(
8237                    selected.provider.clone(),
8238                    selected.model.clone(),
8239                    None,
8240                )),
8241                runner: None,
8242            })
8243            .await
8244            .unwrap()
8245            .thread_id;
8246        let mut rx = runtime.subscribe_events();
8247        let turn_id = runtime
8248            .start_turn(StartTurnRequest {
8249                thread_id,
8250                message: "use selected manual model".to_string(),
8251                images: Vec::new(),
8252                provider_override: None,
8253                model_override: None,
8254                reasoning_override: None,
8255                workspace: test_workspace(),
8256                instructions: InstructionBundle::default(),
8257                developer_context: None,
8258                task_ledger_required: false,
8259                service_tier_override: None,
8260            })
8261            .await
8262            .unwrap();
8263
8264        let mut saw_routing_event = false;
8265        tokio::time::timeout(std::time::Duration::from_secs(5), async {
8266            loop {
8267                let envelope = rx.recv().await.unwrap();
8268                if envelope.turn_id.as_deref() != Some(&turn_id) {
8269                    continue;
8270                }
8271                match envelope.event {
8272                    RoderEvent::InferenceRoutingDecision(_) => {
8273                        saw_routing_event = true;
8274                    }
8275                    RoderEvent::TurnCompleted(_) => break,
8276                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8277                    _ => {}
8278                }
8279            }
8280        })
8281        .await
8282        .unwrap();
8283
8284        assert!(!saw_routing_event);
8285        assert!(contexts.lock().unwrap().is_empty());
8286        let requests = requests.lock().unwrap();
8287        assert_eq!(requests.len(), 1);
8288        assert_eq!(requests[0].model, selected);
8289        let _ = std::fs::remove_dir_all(thread_root);
8290    }
8291
8292    #[test]
8293    fn enabled_inference_router_requires_registered_router() {
8294        let mut builder = ExtensionRegistryBuilder::new();
8295        builder.inference_engine(Arc::new(FakeInferenceEngine));
8296
8297        let err = match Runtime::new(
8298            builder.build().unwrap(),
8299            RuntimeConfig {
8300                inference_router: RuntimeInferenceRouterConfig {
8301                    enabled: true,
8302                    router_id: Some("missing-router".to_string()),
8303                },
8304                ..RuntimeConfig::default()
8305            },
8306        ) {
8307            Ok(_) => panic!("runtime should reject unknown inference router"),
8308            Err(err) => err,
8309        };
8310
8311        assert!(
8312            err.to_string()
8313                .contains("inference router \"missing-router\" is not registered")
8314        );
8315    }
8316
8317    #[tokio::test]
8318    async fn model_profile_routes_request_knobs_to_next_inference() {
8319        let request = captured_profile_request(RuntimeConfig {
8320            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8321            default_model: "gpt-5.5".to_string(),
8322            model_profiles: std::collections::HashMap::from([(
8323                "gpt-5.5".to_string(),
8324                test_model_profile("gpt-5.5"),
8325            )]),
8326            ..RuntimeConfig::default()
8327        })
8328        .await;
8329
8330        let tool_names = request
8331            .tools
8332            .iter()
8333            .map(|tool| tool.name.as_str())
8334            .collect::<Vec<_>>();
8335        assert!(tool_names.contains(&"apply_patch"));
8336        assert!(tool_names.contains(&"edit"));
8337        assert!(tool_names.contains(&"multi_edit"));
8338        assert!(tool_names.contains(&"write_file"));
8339        assert_eq!(request.reasoning.level.as_deref(), Some(REASONING_LOW));
8340        assert_eq!(request.runtime.parallel_tool_calls, Some(false));
8341        assert_eq!(request.runtime.auto_compact_token_limit, Some(123_000));
8342        assert!(
8343            request
8344                .instructions
8345                .developer
8346                .as_deref()
8347                .unwrap_or_default()
8348                .contains("Use the provided context as the current working set")
8349        );
8350        assert_eq!(
8351            request
8352                .metadata
8353                .pointer("/modelProfile/schemaPolicy")
8354                .and_then(serde_json::Value::as_str),
8355            Some("standard_required_first")
8356        );
8357    }
8358
8359    #[tokio::test]
8360    async fn ultra_reasoning_reaches_transport_state_and_enables_proactive_delegation() {
8361        let request = captured_profile_request(RuntimeConfig {
8362            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8363            default_model: "gpt-5.6-sol".to_string(),
8364            reasoning: Some(REASONING_ULTRA.to_string()),
8365            ..RuntimeConfig::default()
8366        })
8367        .await;
8368
8369        assert_eq!(request.reasoning.level.as_deref(), Some(REASONING_ULTRA));
8370        let developer = request
8371            .instructions
8372            .developer
8373            .as_deref()
8374            .expect("ultra developer instructions");
8375        assert!(developer.contains("Proactive multi-agent delegation is active"));
8376    }
8377
8378    #[tokio::test]
8379    async fn lower_sol_effort_requires_explicit_multi_agent_request() {
8380        let request = captured_profile_request(RuntimeConfig {
8381            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8382            default_model: "gpt-5.6-sol".to_string(),
8383            reasoning: Some(roder_api::catalog::REASONING_MEDIUM.to_string()),
8384            ..RuntimeConfig::default()
8385        })
8386        .await;
8387
8388        let developer = request
8389            .instructions
8390            .developer
8391            .as_deref()
8392            .expect("sol developer instructions");
8393        assert!(developer.contains("Do not spawn sub-agents unless"));
8394        assert!(!developer.contains("Proactive multi-agent delegation is active"));
8395    }
8396
8397    #[tokio::test]
8398    async fn luna_does_not_receive_codex_v2_multi_agent_policy() {
8399        let request = captured_profile_request(RuntimeConfig {
8400            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8401            default_model: "gpt-5.6-luna".to_string(),
8402            reasoning: Some(roder_api::catalog::REASONING_MAX.to_string()),
8403            ..RuntimeConfig::default()
8404        })
8405        .await;
8406
8407        let developer = request
8408            .instructions
8409            .developer
8410            .as_deref()
8411            .unwrap_or_default();
8412        assert!(!developer.contains("Do not spawn sub-agents unless"));
8413        assert!(!developer.contains("Proactive multi-agent delegation is active"));
8414    }
8415
8416    #[tokio::test]
8417    async fn ultra_mode_enables_proactive_multi_agent_for_any_model() {
8418        // Luna has no Ultra effort, but ultra mode still injects proactive policy.
8419        let request = captured_profile_request(RuntimeConfig {
8420            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8421            default_model: "gpt-5.6-luna".to_string(),
8422            reasoning: Some(roder_api::catalog::REASONING_MAX.to_string()),
8423            ultra_mode: true,
8424            ..RuntimeConfig::default()
8425        })
8426        .await;
8427
8428        let developer = request
8429            .instructions
8430            .developer
8431            .as_deref()
8432            .expect("ultra mode developer instructions");
8433        assert!(developer.contains("Proactive multi-agent delegation is active"));
8434        assert!(developer.contains("spawn_agent"));
8435    }
8436
8437    #[tokio::test]
8438    async fn ultra_mode_overrides_explicit_request_only_on_sol() {
8439        // Medium Sol effort is normally explicit-request-only; ultra mode flips
8440        // it to proactive without requiring Ultra reasoning effort.
8441        let request = captured_profile_request(RuntimeConfig {
8442            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8443            default_model: "gpt-5.6-sol".to_string(),
8444            reasoning: Some(roder_api::catalog::REASONING_MEDIUM.to_string()),
8445            ultra_mode: true,
8446            ..RuntimeConfig::default()
8447        })
8448        .await;
8449
8450        let developer = request
8451            .instructions
8452            .developer
8453            .as_deref()
8454            .expect("ultra mode sol developer instructions");
8455        assert!(developer.contains("Proactive multi-agent delegation is active"));
8456        assert!(!developer.contains("Do not spawn sub-agents unless"));
8457    }
8458
8459    #[tokio::test]
8460    async fn turn_developer_context_reaches_inference_and_does_not_persist() {
8461        let captured = Arc::new(StdMutex::new(None));
8462        let mut builder = ExtensionRegistryBuilder::new();
8463        builder.inference_engine(Arc::new(CapturingEngine {
8464            request: captured.clone(),
8465        }));
8466        let runtime = Arc::new(
8467            Runtime::new(
8468                builder.build().unwrap(),
8469                RuntimeConfig {
8470                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8471                    default_model: "gpt-5.5".to_string(),
8472                    ..RuntimeConfig::default()
8473                },
8474            )
8475            .unwrap(),
8476        );
8477
8478        async fn run_turn(runtime: &Arc<Runtime>, developer_context: Option<String>) {
8479            let mut rx = runtime.subscribe_events();
8480            let turn_id = runtime
8481                .start_turn(StartTurnRequest {
8482                    thread_id: "thread-turn-context".to_string(),
8483                    message: "hello".to_string(),
8484                    images: Vec::new(),
8485                    provider_override: None,
8486                    model_override: None,
8487                    reasoning_override: None,
8488                    workspace: test_workspace(),
8489                    instructions: InstructionBundle::default(),
8490                    developer_context,
8491                    task_ledger_required: false,
8492                    service_tier_override: None,
8493                })
8494                .await
8495                .unwrap();
8496            tokio::time::timeout(std::time::Duration::from_secs(5), async {
8497                loop {
8498                    let envelope = rx.recv().await.unwrap();
8499                    if envelope.turn_id.as_deref() != Some(&turn_id) {
8500                        continue;
8501                    }
8502                    match envelope.event {
8503                        RoderEvent::TurnCompleted(_) => break,
8504                        RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8505                        _ => {}
8506                    }
8507                }
8508            })
8509            .await
8510            .unwrap();
8511        }
8512
8513        run_turn(
8514            &runtime,
8515            Some("Connected accounts: example-service.".to_string()),
8516        )
8517        .await;
8518        let request = captured.lock().unwrap().clone().unwrap();
8519        assert_eq!(
8520            request.instructions.developer_context.as_deref(),
8521            Some("Connected accounts: example-service.")
8522        );
8523
8524        // The context is per-turn only: the next turn on the same thread
8525        // without a developerContext must not see the previous one.
8526        run_turn(&runtime, None).await;
8527        let request = captured.lock().unwrap().clone().unwrap();
8528        assert_eq!(request.instructions.developer_context, None);
8529    }
8530
8531    #[tokio::test]
8532    async fn tool_search_overrides_route_to_next_inference_request() {
8533        let request = captured_profile_request(RuntimeConfig {
8534            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8535            default_model: "gpt-5.4".to_string(),
8536            tool_search: ToolSearchConfig {
8537                mode: roder_api::inference::ToolSearchMode::Auto,
8538                max_catalog_items: Some(100),
8539                ..ToolSearchConfig::default()
8540            },
8541            provider_tool_search: std::collections::HashMap::from([(
8542                roder_api::catalog::PROVIDER_MOCK.to_string(),
8543                roder_api::inference::ToolSearchConfigOverlay {
8544                    include_skills: Some(false),
8545                    provider_variant: Some(roder_api::inference::ToolSearchProviderVariant::Regex),
8546                    ..Default::default()
8547                },
8548            )]),
8549            model_tool_search: std::collections::HashMap::from([(
8550                "gpt-5.4".to_string(),
8551                roder_api::inference::ToolSearchConfigOverlay {
8552                    mode: Some(roder_api::inference::ToolSearchMode::ProviderNative),
8553                    max_catalog_items: Some(25),
8554                    provider_variant: Some(roder_api::inference::ToolSearchProviderVariant::Bm25),
8555                    ..Default::default()
8556                },
8557            )]),
8558            ..RuntimeConfig::default()
8559        })
8560        .await;
8561
8562        assert_eq!(
8563            request.runtime.tool_search.mode,
8564            roder_api::inference::ToolSearchMode::ProviderNative
8565        );
8566        assert_eq!(request.runtime.tool_search.max_catalog_items, Some(25));
8567        assert_eq!(
8568            request.runtime.tool_search.provider_variant,
8569            roder_api::inference::ToolSearchProviderVariant::Bm25
8570        );
8571    }
8572
8573    #[tokio::test]
8574    async fn context_entrypoint_hints_use_turn_workspace() {
8575        let process_workspace = runtime_test_workspace("entrypoint-process");
8576        let thread_workspace = runtime_test_workspace("entrypoint-thread");
8577        std::fs::create_dir_all(process_workspace.join("src")).unwrap();
8578        std::fs::create_dir_all(thread_workspace.join("src")).unwrap();
8579        std::fs::write(
8580            process_workspace.join("src/sidebar-thread-groups.ts"),
8581            "export const desktopLeak = true;\n",
8582        )
8583        .unwrap();
8584        std::fs::write(
8585            thread_workspace.join("src/voice-plan-feedback.ts"),
8586            "export const voicePlanFeedback = true;\n",
8587        )
8588        .unwrap();
8589
8590        let captured = Arc::new(StdMutex::new(None));
8591        let mut builder = ExtensionRegistryBuilder::new();
8592        builder.inference_engine(Arc::new(CapturingEngine {
8593            request: captured.clone(),
8594        }));
8595        builder.context_planner(Arc::new(roder_context::EntrypointContextPlanner::new(
8596            process_workspace.clone(),
8597        )));
8598        let runtime = Arc::new(
8599            Runtime::new(
8600                builder.build().unwrap(),
8601                RuntimeConfig {
8602                    workspace: Some(process_workspace.display().to_string()),
8603                    ..RuntimeConfig::default()
8604                },
8605            )
8606            .unwrap(),
8607        );
8608        let mut rx = runtime.subscribe_events();
8609
8610        let turn_id = runtime
8611            .start_turn(StartTurnRequest {
8612                thread_id: "thread-workspace-entrypoint".to_string(),
8613                message: "investigate voice plan feedback".to_string(),
8614                images: Vec::new(),
8615                provider_override: None,
8616                model_override: None,
8617                reasoning_override: None,
8618                workspace: thread_workspace.display().to_string(),
8619                instructions: crate::instructions::default_instructions(),
8620                developer_context: None,
8621                task_ledger_required: false,
8622                service_tier_override: None,
8623            })
8624            .await
8625            .unwrap();
8626
8627        tokio::time::timeout(std::time::Duration::from_secs(5), async {
8628            loop {
8629                let envelope = rx.recv().await.unwrap();
8630                if envelope.turn_id.as_deref() != Some(&turn_id) {
8631                    continue;
8632                }
8633                match envelope.event {
8634                    RoderEvent::TurnCompleted(_) => break,
8635                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8636                    _ => {}
8637                }
8638            }
8639        })
8640        .await
8641        .unwrap();
8642
8643        let request = captured.lock().unwrap().clone().expect("captured request");
8644        let transcript_text = request
8645            .transcript
8646            .iter()
8647            .map(|item| match item {
8648                TranscriptItem::UserMessage(message) => message.text.as_str(),
8649                _ => "",
8650            })
8651            .collect::<Vec<_>>()
8652            .join("\n");
8653        assert!(transcript_text.contains("src/voice-plan-feedback.ts"));
8654        assert!(!transcript_text.contains("src/sidebar-thread-groups.ts"));
8655
8656        let _ = std::fs::remove_dir_all(process_workspace);
8657        let _ = std::fs::remove_dir_all(thread_workspace);
8658    }
8659
8660    fn runtime_test_workspace(name: &str) -> std::path::PathBuf {
8661        let path =
8662            std::env::temp_dir().join(format!("roder-runtime-{name}-{}", uuid::Uuid::new_v4()));
8663        let _ = std::fs::remove_dir_all(&path);
8664        std::fs::create_dir_all(&path).unwrap();
8665        path
8666    }
8667
8668    #[tokio::test]
8669    async fn model_profile_user_model_knobs_override_profile_defaults() {
8670        let request = captured_profile_request(RuntimeConfig {
8671            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8672            default_model: "gpt-5.5".to_string(),
8673            reasoning: Some(REASONING_HIGH.to_string()),
8674            auto_compact_token_limit: Some(999),
8675            model_edit_tools: std::collections::HashMap::from([(
8676                "gpt-5.5".to_string(),
8677                EDIT_TOOL_PATCH.to_string(),
8678            )]),
8679            model_parallel_tool_calls: std::collections::HashMap::from([(
8680                "gpt-5.5".to_string(),
8681                true,
8682            )]),
8683            model_profiles: std::collections::HashMap::from([(
8684                "gpt-5.5".to_string(),
8685                test_model_profile("gpt-5.5"),
8686            )]),
8687            ..RuntimeConfig::default()
8688        })
8689        .await;
8690
8691        let tool_names = request
8692            .tools
8693            .iter()
8694            .map(|tool| tool.name.as_str())
8695            .collect::<Vec<_>>();
8696        assert!(tool_names.contains(&"apply_patch"));
8697        assert!(!tool_names.contains(&"edit"));
8698        assert_eq!(request.reasoning.level.as_deref(), Some(REASONING_HIGH));
8699        assert_eq!(request.runtime.parallel_tool_calls, Some(true));
8700        assert_eq!(request.runtime.auto_compact_token_limit, Some(999));
8701    }
8702
8703    #[tokio::test]
8704    async fn runtime_tool_allowlist_filters_advertised_tools() {
8705        let request = captured_profile_request(RuntimeConfig {
8706            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8707            default_model: "gpt-5.5".to_string(),
8708            tool_allowlist: vec!["edit".to_string()],
8709            model_profiles: std::collections::HashMap::from([(
8710                "gpt-5.5".to_string(),
8711                test_model_profile("gpt-5.5"),
8712            )]),
8713            ..RuntimeConfig::default()
8714        })
8715        .await;
8716
8717        let tool_names = request
8718            .tools
8719            .iter()
8720            .map(|tool| tool.name.as_str())
8721            .collect::<Vec<_>>();
8722        assert_eq!(tool_names, vec!["edit"]);
8723    }
8724
8725    /// Builds a store-backed runtime turn for a thread created with the given per-thread
8726    /// overrides and returns the captured inference request.
8727    async fn captured_thread_override_request(
8728        runtime: &Arc<Runtime>,
8729        requests: &Arc<StdMutex<Vec<AgentInferenceRequest>>>,
8730        tool_allowlist: Vec<String>,
8731        developer_instructions: Option<String>,
8732        external_tools: Vec<ToolSpec>,
8733    ) -> AgentInferenceRequest {
8734        let thread_id = runtime
8735            .create_thread_with(CreateThreadRequest {
8736                title: Some("Thread overrides".to_string()),
8737                workspace: test_workspace(),
8738                workspace_id: None,
8739                root_id: None,
8740                provider: None,
8741                model: None,
8742                selection_mode: None,
8743                tool_allowlist,
8744                developer_instructions,
8745                external_tools,
8746                runner: None,
8747            })
8748            .await
8749            .unwrap()
8750            .thread_id;
8751        let mut rx = runtime.subscribe_events();
8752        let turn_id = runtime
8753            .start_turn(StartTurnRequest {
8754                thread_id,
8755                message: "hello".to_string(),
8756                images: Vec::new(),
8757                provider_override: None,
8758                model_override: None,
8759                reasoning_override: None,
8760                workspace: test_workspace(),
8761                instructions: crate::default_instructions(),
8762                developer_context: None,
8763                task_ledger_required: false,
8764                service_tier_override: None,
8765            })
8766            .await
8767            .unwrap();
8768        tokio::time::timeout(std::time::Duration::from_secs(5), async {
8769            loop {
8770                let envelope = rx.recv().await.unwrap();
8771                if envelope.turn_id.as_deref() != Some(&turn_id) {
8772                    continue;
8773                }
8774                match envelope.event {
8775                    RoderEvent::TurnCompleted(_) => break,
8776                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8777                    _ => {}
8778                }
8779            }
8780        })
8781        .await
8782        .unwrap();
8783        requests.lock().unwrap().pop().expect("captured request")
8784    }
8785
8786    #[tokio::test]
8787    async fn thread_tool_allowlist_filters_only_that_thread() {
8788        let requests = Arc::new(StdMutex::new(Vec::new()));
8789        let thread_root =
8790            std::env::temp_dir().join(format!("roder-thread-allowlist-{}", uuid::Uuid::new_v4()));
8791        let mut builder = ExtensionRegistryBuilder::new();
8792        builder.inference_engine(Arc::new(SwitchCaptureEngine {
8793            requests: requests.clone(),
8794        }));
8795        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8796            base_path: thread_root.clone(),
8797        }));
8798        builder.tool_contributor(Arc::new(ProfileToolContributor));
8799        let runtime = Arc::new(
8800            Runtime::new(
8801                builder.build().unwrap(),
8802                RuntimeConfig {
8803                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8804                    default_model: "gpt-5.5".to_string(),
8805                    model_profiles: std::collections::HashMap::from([(
8806                        "gpt-5.5".to_string(),
8807                        test_model_profile("gpt-5.5"),
8808                    )]),
8809                    ..RuntimeConfig::default()
8810                },
8811            )
8812            .unwrap(),
8813        );
8814
8815        let allowlisted = captured_thread_override_request(
8816            &runtime,
8817            &requests,
8818            vec!["edit".to_string()],
8819            None,
8820            Vec::new(),
8821        )
8822        .await;
8823        let unrestricted =
8824            captured_thread_override_request(&runtime, &requests, Vec::new(), None, Vec::new())
8825                .await;
8826
8827        let allowlisted_names = allowlisted
8828            .tools
8829            .iter()
8830            .map(|tool| tool.name.as_str())
8831            .collect::<Vec<_>>();
8832        assert_eq!(allowlisted_names, vec!["edit"]);
8833        let unrestricted_names = unrestricted
8834            .tools
8835            .iter()
8836            .map(|tool| tool.name.as_str())
8837            .collect::<Vec<_>>();
8838        assert!(unrestricted_names.contains(&"edit"));
8839        assert!(unrestricted_names.len() > 1);
8840
8841        let _ = std::fs::remove_dir_all(thread_root);
8842    }
8843
8844    /// Builds a store-backed runtime whose `RuntimeConfig.tool_allowlist` is `["edit"]`.
8845    fn runtime_with_edit_allowlist(
8846        requests: &Arc<StdMutex<Vec<AgentInferenceRequest>>>,
8847        thread_root: &std::path::Path,
8848    ) -> Arc<Runtime> {
8849        let mut builder = ExtensionRegistryBuilder::new();
8850        builder.inference_engine(Arc::new(SwitchCaptureEngine {
8851            requests: requests.clone(),
8852        }));
8853        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8854            base_path: thread_root.to_path_buf(),
8855        }));
8856        builder.tool_contributor(Arc::new(ProfileToolContributor));
8857        Arc::new(
8858            Runtime::new(
8859                builder.build().unwrap(),
8860                RuntimeConfig {
8861                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8862                    default_model: "gpt-5.5".to_string(),
8863                    tool_allowlist: vec!["edit".to_string()],
8864                    model_profiles: std::collections::HashMap::from([(
8865                        "gpt-5.5".to_string(),
8866                        test_model_profile("gpt-5.5"),
8867                    )]),
8868                    ..RuntimeConfig::default()
8869                },
8870            )
8871            .unwrap(),
8872        )
8873    }
8874
8875    #[tokio::test]
8876    async fn runtime_and_thread_allowlists_intersect() {
8877        let requests = Arc::new(StdMutex::new(Vec::new()));
8878        let thread_root = std::env::temp_dir().join(format!(
8879            "roder-allowlist-intersect-{}",
8880            uuid::Uuid::new_v4()
8881        ));
8882        let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8883
8884        let request = captured_thread_override_request(
8885            &runtime,
8886            &requests,
8887            vec!["edit".to_string(), "write_file".to_string()],
8888            None,
8889            Vec::new(),
8890        )
8891        .await;
8892
8893        // The thread allowlist must not re-enable tools the runtime allowlist bans.
8894        let names = request
8895            .tools
8896            .iter()
8897            .map(|tool| tool.name.as_str())
8898            .collect::<Vec<_>>();
8899        assert_eq!(names, vec!["edit"]);
8900
8901        let _ = std::fs::remove_dir_all(thread_root);
8902    }
8903
8904    #[tokio::test]
8905    async fn route_tool_call_denies_tools_outside_allowlists() {
8906        let requests = Arc::new(StdMutex::new(Vec::new()));
8907        let thread_root =
8908            std::env::temp_dir().join(format!("roder-allowlist-dispatch-{}", uuid::Uuid::new_v4()));
8909        let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8910        let thread_id = runtime
8911            .create_thread_with(CreateThreadRequest {
8912                title: Some("Dispatch allowlist".to_string()),
8913                workspace: test_workspace(),
8914                workspace_id: None,
8915                root_id: None,
8916                provider: None,
8917                model: None,
8918                selection_mode: None,
8919                tool_allowlist: vec!["edit".to_string(), "write_file".to_string()],
8920                developer_instructions: None,
8921                external_tools: Vec::new(),
8922                runner: None,
8923            })
8924            .await
8925            .unwrap()
8926            .thread_id;
8927
8928        // write_file is registered and on the thread allowlist but banned by the runtime allowlist.
8929        let result = runtime
8930            .route_tool_call(
8931                &thread_id,
8932                &"turn-allowlist-dispatch".to_string(),
8933                roder_api::inference::ToolCallCompleted {
8934                    id: "call-1".to_string(),
8935                    name: "write_file".to_string(),
8936                    arguments: r#"{"path":"a.txt","content":"hi"}"#.to_string(),
8937                },
8938                None,
8939                None,
8940            )
8941            .await
8942            .unwrap();
8943
8944        assert!(result.is_error);
8945        assert!(
8946            result
8947                .result
8948                .contains("not permitted by the tool allowlist"),
8949            "unexpected result: {}",
8950            result.result
8951        );
8952
8953        let _ = std::fs::remove_dir_all(thread_root);
8954    }
8955
8956    #[tokio::test]
8957    async fn route_tool_calls_denies_agent_swarm_mixed_with_other_tools() {
8958        let requests = Arc::new(StdMutex::new(Vec::new()));
8959        let thread_root =
8960            std::env::temp_dir().join(format!("roder-swarm-exclusive-{}", uuid::Uuid::new_v4()));
8961        let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8962
8963        // A response that mixes agent_swarm with another tool is denied wholesale:
8964        // both calls get an error result with retry guidance, and no tool runs.
8965        let results = runtime
8966            .route_tool_calls(
8967                &"thread-swarm".to_string(),
8968                &"turn-swarm".to_string(),
8969                vec![
8970                    roder_api::inference::ToolCallCompleted {
8971                        id: "swarm-1".to_string(),
8972                        name: "agent_swarm".to_string(),
8973                        arguments: "{}".to_string(),
8974                    },
8975                    roder_api::inference::ToolCallCompleted {
8976                        id: "read-1".to_string(),
8977                        name: "read_file".to_string(),
8978                        arguments: "{}".to_string(),
8979                    },
8980                ],
8981                true,
8982                None,
8983                None,
8984            )
8985            .await
8986            .unwrap();
8987
8988        assert_eq!(results.len(), 2, "every tool_call_id must get a response");
8989        assert_eq!(results[0].id, "swarm-1");
8990        assert_eq!(results[1].id, "read-1");
8991        for result in &results {
8992            assert!(result.is_error);
8993            assert!(
8994                result.result.contains("only tool call"),
8995                "unexpected result: {}",
8996                result.result
8997            );
8998        }
8999
9000        let _ = std::fs::remove_dir_all(thread_root);
9001    }
9002
9003    #[tokio::test]
9004    async fn route_tool_calls_emits_agent_swarm_started_event() {
9005        let requests = Arc::new(StdMutex::new(Vec::new()));
9006        let thread_root =
9007            std::env::temp_dir().join(format!("roder-swarm-started-{}", uuid::Uuid::new_v4()));
9008        let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
9009        let mut events = runtime.subscribe_events();
9010
9011        // A single agent_swarm call (the valid shape) emits AgentSwarmStarted on
9012        // the event bus before dispatch, with the child count parsed from args.
9013        let _ = runtime
9014            .route_tool_calls(
9015                &"thread-swarm".to_string(),
9016                &"turn-swarm".to_string(),
9017                vec![roder_api::inference::ToolCallCompleted {
9018                    id: "swarm-1".to_string(),
9019                    name: "agent_swarm".to_string(),
9020                    arguments: r#"{"description":"x","prompt_template":"Read {{item}}","items":["a.rs","b.rs"]}"#
9021                        .to_string(),
9022                }],
9023                true,
9024                None,
9025                None,
9026            )
9027            .await
9028            .unwrap();
9029
9030        let mut started_child_count = None;
9031        for _ in 0..16 {
9032            let envelope = tokio::time::timeout(std::time::Duration::from_secs(2), events.recv())
9033                .await
9034                .unwrap()
9035                .unwrap();
9036            if let RoderEvent::AgentSwarmStarted(event) = envelope.event {
9037                started_child_count = Some(event.child_count);
9038                assert_eq!(event.tool_id, "swarm-1");
9039                break;
9040            }
9041        }
9042        assert_eq!(started_child_count, Some(2));
9043
9044        let _ = std::fs::remove_dir_all(thread_root);
9045    }
9046
9047    #[test]
9048    fn agent_swarm_child_count_sums_items_and_resumes() {
9049        assert_eq!(
9050            agent_swarm_child_count(r#"{"description":"x","items":["a","b","c"]}"#),
9051            3
9052        );
9053        assert_eq!(
9054            agent_swarm_child_count(
9055                r#"{"description":"x","items":["a"],"resume_agent_ids":{"id1":"continue"}}"#
9056            ),
9057            2
9058        );
9059        // Malformed input is lenient.
9060        assert_eq!(agent_swarm_child_count("not json"), 0);
9061    }
9062
9063    #[test]
9064    fn parse_swarm_counts_reads_summary_with_omitted_buckets() {
9065        let text = "<agent_swarm_result>\n<summary>completed: 2, failed: 1</summary>\n</agent_swarm_result>";
9066        assert_eq!(parse_swarm_counts(text), Some((2, 1, 0)));
9067        let text = "<agent_swarm_result>\n<summary>completed: 0</summary>\n</agent_swarm_result>";
9068        assert_eq!(parse_swarm_counts(text), Some((0, 0, 0)));
9069        // Not a swarm result.
9070        assert_eq!(parse_swarm_counts("just text"), None);
9071    }
9072
9073    /// Signals when inference starts, then waits for `proceed` and fails the stream.
9074    struct SignalledFailureEngine {
9075        started: tokio::sync::mpsc::UnboundedSender<()>,
9076        proceed: Arc<tokio::sync::Notify>,
9077    }
9078
9079    #[async_trait::async_trait]
9080    impl InferenceEngine for SignalledFailureEngine {
9081        fn id(&self) -> String {
9082            roder_api::catalog::PROVIDER_MOCK.to_string()
9083        }
9084
9085        fn capabilities(&self) -> InferenceCapabilities {
9086            InferenceCapabilities::coding_agent_default()
9087        }
9088
9089        async fn list_models(
9090            &self,
9091            _ctx: InferenceProviderContext<'_>,
9092        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
9093            Ok(roder_api::catalog::models_for_provider(
9094                roder_api::catalog::PROVIDER_MOCK,
9095                true,
9096            ))
9097        }
9098
9099        async fn stream_turn(
9100            &self,
9101            _ctx: InferenceTurnContext<'_>,
9102            _request: AgentInferenceRequest,
9103        ) -> anyhow::Result<InferenceEventStream> {
9104            let _ = self.started.send(());
9105            self.proceed.notified().await;
9106            anyhow::bail!("engine failed mid-turn")
9107        }
9108    }
9109
9110    #[tokio::test]
9111    async fn failed_turn_sweeps_pending_external_tool_calls() {
9112        let (started_tx, mut started_rx) = tokio::sync::mpsc::unbounded_channel();
9113        let proceed = Arc::new(tokio::sync::Notify::new());
9114        let mut builder = ExtensionRegistryBuilder::new();
9115        builder.inference_engine(Arc::new(SignalledFailureEngine {
9116            started: started_tx,
9117            proceed: proceed.clone(),
9118        }));
9119        let runtime =
9120            Arc::new(Runtime::new(builder.build().unwrap(), RuntimeConfig::default()).unwrap());
9121        let mut rx = runtime.subscribe_events();
9122        let turn_id = runtime
9123            .start_turn(StartTurnRequest {
9124                thread_id: "thread-sweep".to_string(),
9125                message: "go".to_string(),
9126                images: Vec::new(),
9127                provider_override: None,
9128                model_override: None,
9129                reasoning_override: None,
9130                workspace: test_workspace(),
9131                instructions: InstructionBundle {
9132                    system: None,
9133                    developer: None,
9134                    developer_context: None,
9135                },
9136                developer_context: None,
9137                task_ledger_required: false,
9138                service_tier_override: None,
9139            })
9140            .await
9141            .unwrap();
9142        tokio::time::timeout(std::time::Duration::from_secs(5), started_rx.recv())
9143            .await
9144            .unwrap()
9145            .unwrap();
9146
9147        let (tx, _pending_rx) = oneshot::channel();
9148        runtime.pending_external_tool_calls.lock().await.insert(
9149            "exttool-sweep-test".to_string(),
9150            PendingExternalToolCall {
9151                thread_id: "thread-sweep".to_string(),
9152                turn_id: turn_id.clone(),
9153                tool_id: "call-1".to_string(),
9154                tool_name: "acme_lookup".to_string(),
9155                tx,
9156            },
9157        );
9158        proceed.notify_one();
9159
9160        let outcome = tokio::time::timeout(std::time::Duration::from_secs(5), async {
9161            loop {
9162                let envelope = rx.recv().await.unwrap();
9163                if let RoderEvent::ExternalToolCallResolved(event) = envelope.event
9164                    && event.request_id == "exttool-sweep-test"
9165                {
9166                    break event.outcome;
9167                }
9168            }
9169        })
9170        .await
9171        .expect("turn failure must resolve pending external tool calls");
9172        assert_eq!(outcome, ExternalToolCallOutcome::Cancelled);
9173        assert!(runtime.pending_external_tool_calls.lock().await.is_empty());
9174    }
9175
9176    #[tokio::test]
9177    async fn thread_developer_instructions_layer_under_harness_prompt() {
9178        let requests = Arc::new(StdMutex::new(Vec::new()));
9179        let thread_root = std::env::temp_dir().join(format!(
9180            "roder-thread-instructions-{}",
9181            uuid::Uuid::new_v4()
9182        ));
9183        let mut builder = ExtensionRegistryBuilder::new();
9184        builder.inference_engine(Arc::new(SwitchCaptureEngine {
9185            requests: requests.clone(),
9186        }));
9187        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
9188            base_path: thread_root.clone(),
9189        }));
9190        builder.tool_contributor(Arc::new(ProfileToolContributor));
9191        let runtime =
9192            Arc::new(Runtime::new(builder.build().unwrap(), RuntimeConfig::default()).unwrap());
9193
9194        let request = captured_thread_override_request(
9195            &runtime,
9196            &requests,
9197            Vec::new(),
9198            Some("You are embedded in a host app.".to_string()),
9199            Vec::new(),
9200        )
9201        .await;
9202
9203        let system = request.instructions.system.expect("system instructions");
9204        assert!(system.starts_with("You are Roder"));
9205        let developer = request
9206            .instructions
9207            .developer
9208            .expect("developer instructions");
9209        assert!(developer.starts_with("You are embedded in a host app."));
9210
9211        let plain =
9212            captured_thread_override_request(&runtime, &requests, Vec::new(), None, Vec::new())
9213                .await;
9214        assert_eq!(plain.instructions.developer, None);
9215
9216        let _ = std::fs::remove_dir_all(thread_root);
9217    }
9218
9219    #[tokio::test]
9220    async fn thread_external_tools_are_advertised_and_shadow_builtins() {
9221        let requests = Arc::new(StdMutex::new(Vec::new()));
9222        let thread_root = std::env::temp_dir().join(format!(
9223            "roder-thread-external-tools-{}",
9224            uuid::Uuid::new_v4()
9225        ));
9226        let mut builder = ExtensionRegistryBuilder::new();
9227        builder.inference_engine(Arc::new(SwitchCaptureEngine {
9228            requests: requests.clone(),
9229        }));
9230        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
9231            base_path: thread_root.clone(),
9232        }));
9233        builder.tool_contributor(Arc::new(ProfileToolContributor));
9234        let runtime =
9235            Arc::new(Runtime::new(builder.build().unwrap(), RuntimeConfig::default()).unwrap());
9236
9237        let external_tools = vec![
9238            ToolSpec {
9239                name: "acme_lookup".to_string(),
9240                description: "Look up Acme workspace state.".to_string(),
9241                parameters: serde_json::json!({
9242                    "type": "object",
9243                    "properties": { "query": { "type": "string" } },
9244                    "required": ["query"]
9245                }),
9246            },
9247            ToolSpec {
9248                name: "edit".to_string(),
9249                description: "Host-managed edit.".to_string(),
9250                parameters: serde_json::json!({ "type": "object" }),
9251            },
9252        ];
9253        let request =
9254            captured_thread_override_request(&runtime, &requests, Vec::new(), None, external_tools)
9255                .await;
9256
9257        let acme = request
9258            .tools
9259            .iter()
9260            .find(|tool| tool.name == "acme_lookup")
9261            .expect("external tool advertised");
9262        assert_eq!(acme.description, "Look up Acme workspace state.");
9263        assert_eq!(acme.parameters["required"][0], "query");
9264        let edits = request
9265            .tools
9266            .iter()
9267            .filter(|tool| tool.name == "edit")
9268            .collect::<Vec<_>>();
9269        assert_eq!(edits.len(), 1, "external edit shadows the builtin");
9270        assert_eq!(edits[0].description, "Host-managed edit.");
9271
9272        let plain =
9273            captured_thread_override_request(&runtime, &requests, Vec::new(), None, Vec::new())
9274                .await;
9275        assert!(plain.tools.iter().all(|tool| tool.name != "acme_lookup"));
9276        let plain_edit = plain
9277            .tools
9278            .iter()
9279            .find(|tool| tool.name == "edit")
9280            .expect("builtin edit advertised on plain thread");
9281        assert_eq!(plain_edit.description, "edit test tool");
9282
9283        let _ = std::fs::remove_dir_all(thread_root);
9284    }
9285
9286    #[tokio::test]
9287    async fn model_switch_injects_summary_and_records_profile_segments() {
9288        let requests = Arc::new(StdMutex::new(Vec::new()));
9289        let thread_root = std::env::temp_dir().join(format!(
9290            "roder-model-switch-thread-{}",
9291            uuid::Uuid::new_v4()
9292        ));
9293        let mut builder = ExtensionRegistryBuilder::new();
9294        builder.inference_engine(Arc::new(SwitchCaptureEngine {
9295            requests: requests.clone(),
9296        }));
9297        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
9298            base_path: thread_root.clone(),
9299        }));
9300        builder.tool_contributor(Arc::new(ProfileToolContributor));
9301        let mut claude_profile = test_model_profile("claude-haiku-4-5-20251001");
9302        claude_profile.provider_family = ProviderFamily::Anthropic;
9303        claude_profile.edit_tool = Some(EDIT_TOOL_EDIT.to_string());
9304        let runtime = Arc::new(
9305            Runtime::new(
9306                builder.build().unwrap(),
9307                RuntimeConfig {
9308                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9309                    default_model: "gpt-5.5".to_string(),
9310                    model_profiles: std::collections::HashMap::from([
9311                        ("gpt-5.5".to_string(), test_model_profile("gpt-5.5")),
9312                        ("claude-haiku-4-5-20251001".to_string(), claude_profile),
9313                    ]),
9314                    ..RuntimeConfig::default()
9315                },
9316            )
9317            .unwrap(),
9318        );
9319        let thread_id = runtime
9320            .create_thread_with(CreateThreadRequest {
9321                title: Some("Model switch".to_string()),
9322                workspace: test_workspace(),
9323                workspace_id: None,
9324                root_id: None,
9325                provider: None,
9326                model: None,
9327                selection_mode: None,
9328                tool_allowlist: Vec::new(),
9329                developer_instructions: None,
9330                external_tools: Vec::new(),
9331                runner: None,
9332            })
9333            .await
9334            .unwrap()
9335            .thread_id;
9336        let mut rx = runtime.subscribe_events();
9337        for (message, model_override) in [
9338            ("first turn", None),
9339            ("second turn", Some("claude-haiku-4-5-20251001".to_string())),
9340        ] {
9341            let turn_id = runtime
9342                .start_turn(StartTurnRequest {
9343                    thread_id: thread_id.clone(),
9344                    message: message.to_string(),
9345                    images: Vec::new(),
9346                    provider_override: None,
9347                    model_override,
9348                    reasoning_override: None,
9349                    workspace: test_workspace(),
9350                    instructions: InstructionBundle::default(),
9351                    developer_context: None,
9352                    task_ledger_required: false,
9353                    service_tier_override: None,
9354                })
9355                .await
9356                .unwrap();
9357            tokio::time::timeout(std::time::Duration::from_secs(5), async {
9358                loop {
9359                    let envelope = rx.recv().await.unwrap();
9360                    if envelope.turn_id.as_deref() != Some(&turn_id) {
9361                        continue;
9362                    }
9363                    match envelope.event {
9364                        RoderEvent::TurnCompleted(_) => break,
9365                        RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9366                        _ => {}
9367                    }
9368                }
9369            })
9370            .await
9371            .unwrap();
9372        }
9373
9374        let captured = requests.lock().unwrap().clone();
9375        assert_eq!(captured.len(), 2);
9376        assert!(captured[1].transcript.iter().any(|item| {
9377            matches!(
9378                item,
9379                TranscriptItem::UserMessage(message)
9380                    if message.text.starts_with(MODEL_SWITCH_SUMMARY_PREFIX)
9381                        && message.text.contains("previous profile mock/gpt-5.5")
9382                        && message.text.contains("Current profile mock/claude-haiku-4-5-20251001")
9383                        && message.text.contains("Available tools now:")
9384            )
9385        }));
9386
9387        let snapshot = runtime
9388            .thread_store
9389            .as_ref()
9390            .unwrap()
9391            .load_thread(&thread_id)
9392            .await
9393            .unwrap()
9394            .unwrap();
9395        let trace_segments = snapshot
9396            .turns
9397            .iter()
9398            .flat_map(|turn| &turn.items)
9399            .filter(|item| {
9400                matches!(
9401                    item,
9402                    TranscriptItem::ProviderMetadata(value)
9403                        if value.get("kind").and_then(serde_json::Value::as_str)
9404                            == Some(MODEL_PROFILE_TRACE_KIND)
9405                            && value.get("segment").and_then(serde_json::Value::as_str)
9406                                == Some("assistant")
9407                )
9408            })
9409            .count();
9410        assert!(trace_segments >= 2);
9411        let _ = std::fs::remove_dir_all(thread_root);
9412    }
9413
9414    struct CountingTaskTool {
9415        calls: Arc<StdMutex<u32>>,
9416    }
9417
9418    #[async_trait::async_trait]
9419    impl ToolExecutor for CountingTaskTool {
9420        fn spec(&self) -> ToolSpec {
9421            ToolSpec {
9422                name: "task".to_string(),
9423                description: "Dispatch a test subagent.".to_string(),
9424                parameters: serde_json::json!({
9425                    "type": "object",
9426                    "properties": {
9427                        "description": { "type": "string" },
9428                        "prompt": { "type": "string" },
9429                        "parent_deadline_seconds": { "type": "integer" }
9430                    },
9431                    "required": ["description", "prompt"],
9432                    "additionalProperties": false
9433                }),
9434            }
9435        }
9436
9437        async fn execute(
9438            &self,
9439            _ctx: ToolExecutionContext,
9440            call: ToolCall,
9441        ) -> anyhow::Result<ToolResult> {
9442            *self.calls.lock().unwrap() += 1;
9443            Ok(ToolResult {
9444                id: call.id,
9445                name: call.name,
9446                text: "started child".to_string(),
9447                data: serde_json::json!({}),
9448                is_error: false,
9449            })
9450        }
9451    }
9452
9453    #[tokio::test]
9454    async fn runtime_profile_reaches_inference_request_and_turn_metadata() {
9455        let captured = Arc::new(StdMutex::new(None));
9456        let mut builder = ExtensionRegistryBuilder::new();
9457        builder.inference_engine(Arc::new(CapturingEngine {
9458            request: captured.clone(),
9459        }));
9460        let runtime = Arc::new(
9461            Runtime::new(
9462                builder.build().unwrap(),
9463                RuntimeConfig {
9464                    runtime_profile: RuntimeProfile::NonInteractive,
9465                    ..RuntimeConfig::default()
9466                },
9467            )
9468            .unwrap(),
9469        );
9470        let mut rx = runtime.subscribe_events();
9471        let turn_id = runtime
9472            .start_turn(StartTurnRequest {
9473                thread_id: "thread-profile".to_string(),
9474                message: "work unattended".to_string(),
9475                images: Vec::new(),
9476                provider_override: None,
9477                model_override: None,
9478                reasoning_override: None,
9479                workspace: test_workspace(),
9480                instructions: InstructionBundle {
9481                    system: None,
9482                    developer: Some("base developer".to_string()),
9483                    developer_context: None,
9484                },
9485                developer_context: None,
9486                task_ledger_required: false,
9487                service_tier_override: None,
9488            })
9489            .await
9490            .unwrap();
9491
9492        let mut observed_profile = None;
9493        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9494            loop {
9495                let envelope = rx.recv().await.unwrap();
9496                if envelope.turn_id.as_deref() != Some(&turn_id) {
9497                    continue;
9498                }
9499                match envelope.event {
9500                    RoderEvent::TurnStarted(event) => {
9501                        observed_profile = Some(event.runtime_profile);
9502                    }
9503                    RoderEvent::TurnCompleted(_) => break,
9504                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9505                    _ => {}
9506                }
9507            }
9508        })
9509        .await
9510        .unwrap();
9511
9512        assert_eq!(observed_profile, Some(RuntimeProfile::NonInteractive));
9513        let request = captured.lock().unwrap().clone().unwrap();
9514        assert_eq!(request.runtime.profile, RuntimeProfile::NonInteractive);
9515        let developer = request.instructions.developer.unwrap();
9516        assert!(developer.contains("base developer"));
9517        assert!(developer.contains("non-interactive profile"));
9518    }
9519
9520    #[tokio::test]
9521    async fn global_policy_mode_changes_do_not_create_runtime_thread_directory() {
9522        let workspace = runtime_test_workspace("global-policy-mode");
9523        let thread_root = workspace.join("threads");
9524        let mut builder = ExtensionRegistryBuilder::new();
9525        builder.inference_engine(Arc::new(FakeInferenceEngine));
9526        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
9527            base_path: thread_root.clone(),
9528        }));
9529        let runtime = Runtime::new(
9530            builder.build().unwrap(),
9531            RuntimeConfig {
9532                workspace: Some(workspace.display().to_string()),
9533                ..Default::default()
9534            },
9535        )
9536        .unwrap();
9537
9538        runtime
9539            .set_policy_mode(PolicyMode::AcceptAll, Some("test".to_string()))
9540            .await
9541            .unwrap();
9542
9543        assert!(!thread_root.join("runtime").exists());
9544        let _ = std::fs::remove_dir_all(workspace);
9545    }
9546
9547    #[tokio::test]
9548    async fn task_ledger_enforcement_injects_eval_reminder_before_work() {
9549        let captured = Arc::new(StdMutex::new(None));
9550        let mut builder = ExtensionRegistryBuilder::new();
9551        builder.inference_engine(Arc::new(CapturingEngine {
9552            request: captured.clone(),
9553        }));
9554        builder.tool_contributor(Arc::new(
9555            roder_ext_task_ledger::TaskLedgerToolContributor::default(),
9556        ));
9557        let runtime = Arc::new(
9558            Runtime::new(
9559                builder.build().unwrap(),
9560                RuntimeConfig {
9561                    runtime_profile: RuntimeProfile::Eval,
9562                    policy_mode: PolicyMode::Bypass,
9563                    agent_swarm_mode: false,
9564                    ultra_mode: false,
9565                    ..RuntimeConfig::default()
9566                },
9567            )
9568            .unwrap(),
9569        );
9570        let mut rx = runtime.subscribe_events();
9571        let turn_id = runtime
9572            .start_turn(StartTurnRequest {
9573                thread_id: "thread-ledger".to_string(),
9574                message: "decomposed work".to_string(),
9575                images: Vec::new(),
9576                provider_override: None,
9577                model_override: None,
9578                reasoning_override: None,
9579                workspace: test_workspace(),
9580                instructions: InstructionBundle::default(),
9581                developer_context: None,
9582                task_ledger_required: true,
9583                service_tier_override: None,
9584            })
9585            .await
9586            .unwrap();
9587
9588        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9589            loop {
9590                let envelope = rx.recv().await.unwrap();
9591                if envelope.turn_id.as_deref() == Some(&turn_id)
9592                    && matches!(envelope.event, RoderEvent::TurnCompleted(_))
9593                {
9594                    break;
9595                }
9596            }
9597        })
9598        .await
9599        .unwrap();
9600
9601        let request = captured.lock().unwrap().clone().unwrap();
9602        let developer = request.instructions.developer.unwrap();
9603        assert!(developer.contains("Task Ledger Required"));
9604        assert!(developer.contains("task_ledger.update"));
9605        let tool_names: Vec<_> = request
9606            .tools
9607            .iter()
9608            .map(|tool| tool.name.as_str())
9609            .collect();
9610        assert!(
9611            tool_names.contains(&TASK_LEDGER_TOOL_NAME),
9612            "tool names: {tool_names:?}"
9613        );
9614        assert_eq!(
9615            request.tool_choice,
9616            ToolChoice::Specific(TASK_LEDGER_TOOL_NAME.to_string())
9617        );
9618        assert_eq!(request.tools.len(), 1);
9619        assert_eq!(request.tools[0].name, TASK_LEDGER_TOOL_NAME);
9620    }
9621
9622    #[tokio::test]
9623    async fn eval_task_ledger_blocks_final_answer_until_open_items_are_completed() {
9624        let requests = Arc::new(StdMutex::new(Vec::new()));
9625        let mut builder = ExtensionRegistryBuilder::new();
9626        builder.inference_engine(Arc::new(TaskLedgerCompletionGateEngine {
9627            calls: StdMutex::new(0),
9628            requests: requests.clone(),
9629        }));
9630        builder.tool_contributor(Arc::new(
9631            roder_ext_task_ledger::TaskLedgerToolContributor::default(),
9632        ));
9633        let runtime = Arc::new(
9634            Runtime::new(
9635                builder.build().unwrap(),
9636                RuntimeConfig {
9637                    runtime_profile: RuntimeProfile::Eval,
9638                    policy_mode: PolicyMode::Bypass,
9639                    agent_swarm_mode: false,
9640                    ultra_mode: false,
9641                    ..RuntimeConfig::default()
9642                },
9643            )
9644            .unwrap(),
9645        );
9646        let mut rx = runtime.subscribe_events();
9647        let turn_id = runtime
9648            .start_turn(StartTurnRequest {
9649                thread_id: "thread-ledger-completion".to_string(),
9650                message: "write the answer file".to_string(),
9651                images: Vec::new(),
9652                provider_override: None,
9653                model_override: None,
9654                reasoning_override: None,
9655                workspace: test_workspace(),
9656                instructions: InstructionBundle::default(),
9657                developer_context: None,
9658                task_ledger_required: true,
9659                service_tier_override: None,
9660            })
9661            .await
9662            .unwrap();
9663
9664        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9665            loop {
9666                let envelope = rx.recv().await.unwrap();
9667                if envelope.turn_id.as_deref() != Some(&turn_id) {
9668                    continue;
9669                }
9670                match envelope.event {
9671                    RoderEvent::TurnCompleted(_) => break,
9672                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9673                    _ => {}
9674                }
9675            }
9676        })
9677        .await
9678        .unwrap();
9679
9680        let requests = requests.lock().unwrap().clone();
9681        assert_eq!(requests.len(), 4);
9682        assert!(requests[2].transcript.iter().any(|item| {
9683            matches!(
9684                item,
9685                TranscriptItem::UserMessage(message)
9686                    if message.text.contains("Task Ledger Completion Required")
9687                        && message.text.contains("Write /app/result.txt")
9688            )
9689        }));
9690        assert!(requests[3].transcript.iter().any(|item| {
9691            matches!(
9692                item,
9693                TranscriptItem::ToolResult(result)
9694                    if result.name.as_deref() == Some(TASK_LEDGER_TOOL_NAME)
9695                        && result.result.contains("Task ledger: 2/2 completed")
9696            )
9697        }));
9698    }
9699
9700    #[tokio::test]
9701    async fn eval_task_ledger_checkpoint_requests_scoreable_file_before_final_reserve() {
9702        let requests = Arc::new(StdMutex::new(Vec::new()));
9703        let mut builder = ExtensionRegistryBuilder::new();
9704        builder.inference_engine(Arc::new(TaskLedgerCompletionGateEngine {
9705            calls: StdMutex::new(0),
9706            requests: requests.clone(),
9707        }));
9708        builder.tool_contributor(Arc::new(
9709            roder_ext_task_ledger::TaskLedgerToolContributor::default(),
9710        ));
9711        let runtime = Arc::new(
9712            Runtime::new(
9713                builder.build().unwrap(),
9714                RuntimeConfig {
9715                    runtime_profile: RuntimeProfile::Eval,
9716                    policy_mode: PolicyMode::Bypass,
9717                    agent_swarm_mode: false,
9718                    ultra_mode: false,
9719                    turn_deadline_seconds: Some(120),
9720                    ..RuntimeConfig::default()
9721                },
9722            )
9723            .unwrap(),
9724        );
9725        let mut rx = runtime.subscribe_events();
9726        let turn_id = runtime
9727            .start_turn(StartTurnRequest {
9728                thread_id: "thread-ledger-checkpoint".to_string(),
9729                message: "write the answer file".to_string(),
9730                images: Vec::new(),
9731                provider_override: None,
9732                model_override: None,
9733                reasoning_override: None,
9734                workspace: test_workspace(),
9735                instructions: InstructionBundle::default(),
9736                developer_context: None,
9737                task_ledger_required: true,
9738                service_tier_override: None,
9739            })
9740            .await
9741            .unwrap();
9742
9743        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9744            loop {
9745                let envelope = rx.recv().await.unwrap();
9746                if envelope.turn_id.as_deref() != Some(&turn_id) {
9747                    continue;
9748                }
9749                match envelope.event {
9750                    RoderEvent::TurnCompleted(_) => break,
9751                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9752                    _ => {}
9753                }
9754            }
9755        })
9756        .await
9757        .unwrap();
9758
9759        let requests = requests.lock().unwrap().clone();
9760        assert!(requests.len() >= 2);
9761        assert!(requests[1].transcript.iter().any(|item| {
9762            matches!(
9763                item,
9764                TranscriptItem::UserMessage(message)
9765                    if message.text.contains("Scoreable Output Checkpoint")
9766                        && message.text.contains("ensure the required output file(s) exist")
9767                        && message.text.contains("Write /app/result.txt")
9768            )
9769        }));
9770    }
9771
9772    #[test]
9773    fn deadline_task_ledger_prompt_preserves_scoreable_work_instruction() {
9774        let prompt = task_ledger_deadline_completion_prompt(
9775            12,
9776            30,
9777            "Task Ledger Completion Required: write /app/result.txt, then call task_ledger.update",
9778        );
9779
9780        assert!(prompt.contains("12 seconds remain"));
9781        assert!(prompt.contains("create or update the required scoreable output files"));
9782        assert!(prompt.contains("write /app/result.txt"));
9783        assert!(prompt.contains(TASK_LEDGER_TOOL_NAME));
9784    }
9785
9786    #[test]
9787    fn scoreable_checkpoint_prompt_preserves_provisional_file_instruction() {
9788        let prompt = task_ledger_scoreable_checkpoint_prompt(
9789            120,
9790            "Task Ledger Completion Required: write /app/result.txt, then call task_ledger.update",
9791        );
9792
9793        assert!(prompt.contains("120 seconds remain"));
9794        assert!(prompt.contains("best evidence-backed answer"));
9795        assert!(prompt.contains("even if provisional"));
9796        assert!(prompt.contains("preserve that candidate"));
9797        assert!(prompt.contains("partial-coverage"));
9798        assert!(prompt.contains("write /app/result.txt"));
9799        assert!(prompt.contains(TASK_LEDGER_TOOL_NAME));
9800    }
9801
9802    #[test]
9803    fn open_task_ledger_moves_inference_timeout_to_scoreable_checkpoint() {
9804        let deadline = Some(OffsetDateTime::now_utc() + Duration::seconds(870));
9805        let transcript = vec![TranscriptItem::ToolResult(ToolResultRecord {
9806            id: "ledger-open".to_string(),
9807            name: Some(TASK_LEDGER_TOOL_NAME.to_string()),
9808            result: "Task ledger: 0/1 completed\n- pending: Write /app/result.txt [write]"
9809                .to_string(),
9810            display_payload: None,
9811            is_error: false,
9812        })];
9813
9814        let (_, action) = inference_timeout_deadline(
9815            deadline,
9816            RuntimeProfile::Eval,
9817            true,
9818            30,
9819            false,
9820            0,
9821            &transcript,
9822        )
9823        .unwrap();
9824
9825        assert_eq!(action, InferenceTimeoutAction::ScoreableCheckpoint);
9826    }
9827
9828    #[tokio::test]
9829    async fn verification_gate_forces_eval_code_changes_through_review() {
9830        let mut builder = ExtensionRegistryBuilder::new();
9831        builder.inference_engine(Arc::new(VerificationGateEngine {
9832            calls: StdMutex::new(0),
9833        }));
9834        builder.tool_contributor(Arc::new(WriteFileContributor));
9835        builder.tool_contributor(Arc::new(
9836            roder_ext_verification::VerificationToolContributor,
9837        ));
9838        let runtime = Arc::new(
9839            Runtime::new(
9840                builder.build().unwrap(),
9841                RuntimeConfig {
9842                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9843                    default_model: "mock".to_string(),
9844                    runtime_profile: RuntimeProfile::Eval,
9845                    policy_mode: PolicyMode::Bypass,
9846                    agent_swarm_mode: false,
9847                    ultra_mode: false,
9848                    ..RuntimeConfig::default()
9849                },
9850            )
9851            .unwrap(),
9852        );
9853        let mut rx = runtime.subscribe_events();
9854        let turn_id = runtime
9855            .start_turn(StartTurnRequest {
9856                thread_id: "thread-verification".to_string(),
9857                message: "write code".to_string(),
9858                images: Vec::new(),
9859                provider_override: None,
9860                model_override: None,
9861                reasoning_override: None,
9862                workspace: test_workspace(),
9863                instructions: InstructionBundle::default(),
9864                developer_context: None,
9865                task_ledger_required: false,
9866                service_tier_override: None,
9867            })
9868            .await
9869            .unwrap();
9870
9871        let mut saw_required = false;
9872        let mut saw_completed = false;
9873        let mut final_text = String::new();
9874        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9875            loop {
9876                let envelope = rx.recv().await.unwrap();
9877                if envelope.turn_id.as_deref() != Some(&turn_id) {
9878                    continue;
9879                }
9880                match envelope.event {
9881                    RoderEvent::VerificationRequired(event) => {
9882                        saw_required = true;
9883                        assert_eq!(event.changed_files, vec!["src/lib.rs"]);
9884                    }
9885                    RoderEvent::VerificationCompleted(event) => {
9886                        saw_completed = true;
9887                        assert!(event.passed);
9888                    }
9889                    RoderEvent::InferenceEventReceived(event) => {
9890                        if let InferenceEvent::MessageDelta(delta) = event.event {
9891                            final_text.push_str(&delta.text);
9892                        }
9893                    }
9894                    RoderEvent::TurnCompleted(_) => break,
9895                    _ => {}
9896                }
9897            }
9898        })
9899        .await
9900        .unwrap();
9901
9902        assert!(saw_required);
9903        assert!(saw_completed);
9904        assert!(final_text.contains("verified final"));
9905    }
9906
9907    #[tokio::test]
9908    async fn speed_policy_changes_reasoning_across_eval_model_calls_without_model_switch() {
9909        let requests = Arc::new(StdMutex::new(Vec::new()));
9910        let mut builder = ExtensionRegistryBuilder::new();
9911        builder.inference_engine(Arc::new(SpeedPolicyEngine {
9912            calls: StdMutex::new(0),
9913            requests: requests.clone(),
9914        }));
9915        builder.tool_contributor(Arc::new(WriteFileContributor));
9916        builder.tool_contributor(Arc::new(
9917            roder_ext_verification::VerificationToolContributor,
9918        ));
9919        let runtime = Arc::new(
9920            Runtime::new(
9921                builder.build().unwrap(),
9922                RuntimeConfig {
9923                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9924                    default_model: "gpt-5.5".to_string(),
9925                    runtime_profile: RuntimeProfile::Eval,
9926                    policy_mode: PolicyMode::Bypass,
9927                    agent_swarm_mode: false,
9928                    ultra_mode: false,
9929                    ..RuntimeConfig::default()
9930                },
9931            )
9932            .unwrap(),
9933        );
9934        let mut rx = runtime.subscribe_events();
9935        let turn_id = runtime
9936            .start_turn(StartTurnRequest {
9937                thread_id: "thread-speed-policy".to_string(),
9938                message: "write code".to_string(),
9939                images: Vec::new(),
9940                provider_override: None,
9941                model_override: None,
9942                reasoning_override: None,
9943                workspace: test_workspace(),
9944                instructions: InstructionBundle::default(),
9945                developer_context: None,
9946                task_ledger_required: false,
9947                service_tier_override: None,
9948            })
9949            .await
9950            .unwrap();
9951
9952        let mut saw_speed_policy_event = false;
9953        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9954            loop {
9955                let envelope = rx.recv().await.unwrap();
9956                if envelope.turn_id.as_deref() != Some(&turn_id) {
9957                    continue;
9958                }
9959                match envelope.event {
9960                    RoderEvent::InferenceStarted(event) => {
9961                        if event.speed_policy.is_some() {
9962                            saw_speed_policy_event = true;
9963                        }
9964                    }
9965                    RoderEvent::TurnCompleted(_) => break,
9966                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9967                    _ => {}
9968                }
9969            }
9970        })
9971        .await
9972        .unwrap();
9973
9974        let requests = requests.lock().unwrap().clone();
9975        assert!(saw_speed_policy_event);
9976        assert!(requests.len() >= 4);
9977        assert!(requests.iter().all(|request| {
9978            request.model.provider == roder_api::catalog::PROVIDER_MOCK
9979                && request.model.model == "gpt-5.5"
9980        }));
9981        assert_eq!(
9982            requests[0].runtime.speed_policy.as_ref().map(|d| d.phase),
9983            Some(roder_api::inference::SpeedPolicyPhase::Orientation)
9984        );
9985        assert_eq!(requests[0].reasoning.level.as_deref(), Some(REASONING_HIGH));
9986        assert_eq!(
9987            requests[1].runtime.speed_policy.as_ref().map(|d| d.phase),
9988            Some(roder_api::inference::SpeedPolicyPhase::Execution)
9989        );
9990        assert_eq!(requests[1].reasoning.level.as_deref(), Some(REASONING_LOW));
9991        assert_eq!(
9992            requests[2].runtime.speed_policy.as_ref().map(|d| d.phase),
9993            Some(roder_api::inference::SpeedPolicyPhase::Verification)
9994        );
9995        assert_eq!(requests[2].reasoning.level.as_deref(), Some(REASONING_HIGH));
9996        assert_eq!(
9997            requests[2]
9998                .metadata
9999                .pointer("/speedPolicy/phase")
10000                .and_then(serde_json::Value::as_str),
10001            Some("verification")
10002        );
10003    }
10004
10005    #[tokio::test]
10006    async fn deadline_turn_timeout_emits_partial_result_and_clears_active_turn() {
10007        let mut builder = ExtensionRegistryBuilder::new();
10008        builder.inference_engine(Arc::new(DeadlineEngine));
10009        let runtime = Arc::new(
10010            Runtime::new(
10011                builder.build().unwrap(),
10012                RuntimeConfig {
10013                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
10014                    default_model: "mock".to_string(),
10015                    runtime_profile: RuntimeProfile::Eval,
10016                    turn_deadline_seconds: Some(1),
10017                    ..RuntimeConfig::default()
10018                },
10019            )
10020            .unwrap(),
10021        );
10022        let mut rx = runtime.subscribe_events();
10023        let turn_id = runtime
10024            .start_turn(StartTurnRequest {
10025                thread_id: "thread-deadline".to_string(),
10026                message: "slow work".to_string(),
10027                images: Vec::new(),
10028                provider_override: None,
10029                model_override: None,
10030                reasoning_override: None,
10031                workspace: test_workspace(),
10032                instructions: InstructionBundle::default(),
10033                developer_context: None,
10034                task_ledger_required: false,
10035                service_tier_override: None,
10036            })
10037            .await
10038            .unwrap();
10039
10040        let mut saw_partial = false;
10041        let mut saw_deadline = false;
10042        let mut failed_kind = None;
10043        tokio::time::timeout(std::time::Duration::from_secs(5), async {
10044            loop {
10045                let envelope = rx.recv().await.unwrap();
10046                if envelope.turn_id.as_deref() != Some(&turn_id) {
10047                    continue;
10048                }
10049                match envelope.event {
10050                    RoderEvent::TurnPartialResult(event) => {
10051                        saw_partial = event.summary.contains("partial turn state");
10052                    }
10053                    RoderEvent::TurnDeadlineExceeded(event) => {
10054                        saw_deadline = event.partial_result.contains("transcript items");
10055                    }
10056                    RoderEvent::TurnFailed(event) => {
10057                        failed_kind = event.error_kind;
10058                        break;
10059                    }
10060                    _ => {}
10061                }
10062            }
10063        })
10064        .await
10065        .unwrap();
10066
10067        for _ in 0..20 {
10068            if !runtime.active_turns.read().await.contains_key(&turn_id) {
10069                break;
10070            }
10071            tokio::time::sleep(std::time::Duration::from_millis(10)).await;
10072        }
10073        assert!(saw_partial);
10074        assert!(saw_deadline);
10075        assert_eq!(failed_kind.as_deref(), Some("deadline_timeout"));
10076        assert!(!runtime.active_turns.read().await.contains_key(&turn_id));
10077    }
10078
10079    #[tokio::test]
10080    async fn deadline_skips_subagent_task_when_remaining_budget_is_too_low() {
10081        let calls = Arc::new(StdMutex::new(0));
10082        let mut builder = ExtensionRegistryBuilder::new();
10083        builder.inference_engine(Arc::new(CapturingEngine {
10084            request: Arc::new(StdMutex::new(None)),
10085        }));
10086        let task_tool = Arc::new(CountingTaskTool {
10087            calls: calls.clone(),
10088        });
10089        builder.tool_contributor(Arc::new(TestToolContributor { tool: task_tool }));
10090        let runtime = Arc::new(
10091            Runtime::new(
10092                builder.build().unwrap(),
10093                RuntimeConfig {
10094                    policy_mode: PolicyMode::Bypass,
10095                    agent_swarm_mode: false,
10096                    ultra_mode: false,
10097                    ..RuntimeConfig::default()
10098                },
10099            )
10100            .unwrap(),
10101        );
10102
10103        let result = runtime
10104            .route_tool_call(
10105                &"thread-deadline-task".to_string(),
10106                &"turn-deadline-task".to_string(),
10107                ToolCallCompleted {
10108                    id: "task-1".to_string(),
10109                    name: "task".to_string(),
10110                    arguments: serde_json::json!({
10111                        "description": "inspect",
10112                        "prompt": "read"
10113                    })
10114                    .to_string(),
10115                },
10116                None,
10117                Some(OffsetDateTime::now_utc() + Duration::seconds(1)),
10118            )
10119            .await
10120            .unwrap();
10121
10122        assert!(result.is_error);
10123        assert!(result.result.contains("deadline policy skipped"));
10124        assert_eq!(*calls.lock().unwrap(), 0);
10125    }
10126
10127    struct TestToolContributor {
10128        tool: Arc<dyn ToolExecutor>,
10129    }
10130
10131    impl ToolContributor for TestToolContributor {
10132        fn id(&self) -> String {
10133            "test-tool".to_string()
10134        }
10135
10136        fn contribute(&self, registry: &mut ToolRegistry) -> anyhow::Result<()> {
10137            registry.register(self.tool.clone())
10138        }
10139    }
10140
10141    #[tokio::test]
10142    async fn agent_swarm_mode_override_is_per_thread() {
10143        let runtime = Runtime::fake().unwrap();
10144        let trigger = roder_api::subagents::AgentSwarmModeTrigger::Manual;
10145
10146        // Off everywhere by default.
10147        assert!(
10148            !runtime
10149                .effective_agent_swarm_mode_for_thread("thread-a")
10150                .await
10151        );
10152        assert!(
10153            !runtime
10154                .effective_agent_swarm_mode_for_thread("thread-b")
10155                .await
10156        );
10157
10158        // Enabling on thread-a does not leak into thread-b.
10159        assert!(
10160            runtime
10161                .set_agent_swarm_mode_for_thread("thread-a", true, trigger)
10162                .await
10163        );
10164        assert!(
10165            runtime
10166                .effective_agent_swarm_mode_for_thread("thread-a")
10167                .await
10168        );
10169        assert!(
10170            !runtime
10171                .effective_agent_swarm_mode_for_thread("thread-b")
10172                .await
10173        );
10174
10175        // A per-thread `off` override wins over a runtime-global `on` default.
10176        runtime.set_agent_swarm_mode(true, trigger).await.unwrap();
10177        runtime
10178            .set_agent_swarm_mode_for_thread("thread-b", false, trigger)
10179            .await;
10180        assert!(
10181            !runtime
10182                .effective_agent_swarm_mode_for_thread("thread-b")
10183                .await,
10184            "explicit per-thread off overrides the global on default"
10185        );
10186        // A thread with no override still follows the global default.
10187        assert!(
10188            runtime
10189                .effective_agent_swarm_mode_for_thread("thread-c")
10190                .await,
10191            "threads without an override follow the runtime-global default"
10192        );
10193    }
10194
10195    #[tokio::test]
10196    async fn set_agent_swarm_mode_for_thread_emits_event_with_real_thread_id() {
10197        let runtime = Runtime::fake().unwrap();
10198        let mut events = runtime.subscribe_events();
10199        runtime
10200            .set_agent_swarm_mode_for_thread(
10201                "thread-xyz",
10202                true,
10203                roder_api::subagents::AgentSwarmModeTrigger::Task,
10204            )
10205            .await;
10206        let mut saw = false;
10207        for _ in 0..8 {
10208            let envelope = tokio::time::timeout(std::time::Duration::from_secs(2), events.recv())
10209                .await
10210                .unwrap()
10211                .unwrap();
10212            if let RoderEvent::AgentSwarmModeChanged(event) = envelope.event {
10213                assert_eq!(event.thread_id, "thread-xyz");
10214                assert!(event.enabled);
10215                assert_eq!(
10216                    event.trigger,
10217                    roder_api::subagents::AgentSwarmModeTrigger::Task
10218                );
10219                saw = true;
10220                break;
10221            }
10222        }
10223        assert!(
10224            saw,
10225            "expected an AgentSwarmModeChanged event for the thread"
10226        );
10227    }
10228
10229    #[tokio::test]
10230    async fn ultra_mode_override_is_per_thread() {
10231        let runtime = Runtime::fake().unwrap();
10232        let trigger = roder_api::subagents::UltraModeTrigger::Manual;
10233
10234        assert!(!runtime.effective_ultra_mode_for_thread("thread-a").await);
10235        assert!(!runtime.effective_ultra_mode_for_thread("thread-b").await);
10236
10237        assert!(
10238            runtime
10239                .set_ultra_mode_for_thread("thread-a", true, trigger)
10240                .await
10241        );
10242        assert!(runtime.effective_ultra_mode_for_thread("thread-a").await);
10243        assert!(!runtime.effective_ultra_mode_for_thread("thread-b").await);
10244
10245        runtime.set_ultra_mode(true, trigger).await.unwrap();
10246        runtime
10247            .set_ultra_mode_for_thread("thread-b", false, trigger)
10248            .await;
10249        assert!(
10250            !runtime.effective_ultra_mode_for_thread("thread-b").await,
10251            "explicit per-thread off overrides the global on default"
10252        );
10253        assert!(
10254            runtime.effective_ultra_mode_for_thread("thread-c").await,
10255            "threads without an override follow the runtime-global default"
10256        );
10257    }
10258
10259    #[tokio::test]
10260    async fn set_ultra_mode_for_thread_emits_event_with_real_thread_id() {
10261        let runtime = Runtime::fake().unwrap();
10262        let mut events = runtime.subscribe_events();
10263        runtime
10264            .set_ultra_mode_for_thread(
10265                "thread-ultra",
10266                true,
10267                roder_api::subagents::UltraModeTrigger::Task,
10268            )
10269            .await;
10270        let mut saw = false;
10271        for _ in 0..8 {
10272            let envelope = tokio::time::timeout(std::time::Duration::from_secs(2), events.recv())
10273                .await
10274                .unwrap()
10275                .unwrap();
10276            if let RoderEvent::UltraModeChanged(event) = envelope.event {
10277                assert_eq!(event.thread_id, "thread-ultra");
10278                assert!(event.enabled);
10279                assert_eq!(event.trigger, roder_api::subagents::UltraModeTrigger::Task);
10280                saw = true;
10281                break;
10282            }
10283        }
10284        assert!(saw, "expected an UltraModeChanged event for the thread");
10285    }
10286}
10287
10288#[cfg(test)]
10289#[path = "runtime/mailbox_test_helpers.rs"]
10290mod mailbox_test_helpers;