Skip to main content

roder_core/
runtime.rs

1#[path = "runtime/sampling.rs"]
2pub(crate) mod sampling;
3#[path = "tool_advertisement.rs"]
4mod tool_advertisement;
5use sampling::{
6    SamplingPreempted, SamplingRetry, completed_call_replayed, preemptible, wait_for_steer,
7};
8#[path = "runtime/sampling_output.rs"]
9mod sampling_output;
10use sampling_output::{OutputContext, SamplingOutput};
11mod compaction_template;
12#[path = "runtime/eager_tools.rs"]
13mod eager_tools;
14mod thread_admission;
15mod owner_handoff;
16mod execution_receipts;
17use eager_tools::EagerTools;
18#[path = "runtime/patch_progress.rs"]
19mod patch_progress;
20use patch_progress::PatchProgressTracker;
21
22use std::collections::{HashMap, HashSet};
23use std::path::PathBuf;
24use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
25use std::sync::{Arc, Weak};
26
27use anyhow::Context;
28use futures::StreamExt;
29use futures::future::{AbortHandle, Abortable, BoxFuture, try_join_all};
30use roder_api::catalog::{
31    EDIT_TOOL_EDIT, EDIT_TOOL_PATCH, PROVIDER_GEMINI, REASONING_NONE, REASONING_ULTRA,
32    built_in_model_profile, built_in_model_profile_for_provider, lookup_model,
33    model_supports_reasoning_effort,
34};
35use roder_api::context::PolicyGate;
36use roder_api::events::*;
37use roder_api::extension::ExtensionRegistry;
38use roder_api::inference::{
39    AgentInferenceRequest, HostedWebSearchConfig, HostedWebSearchMode, InferenceEngine,
40    InferenceEvent, InferenceTurnContext, InstructionBundle, ModelHarnessProfile,
41    ModelSchemaPolicy, ModelSelection, OutputConfig, ProviderTurnCleanup, ReasoningConfig,
42    RuntimeHints, RuntimeProfile, TokenUsage, ToolCallCompleted, ToolSearchConfig,
43    ToolSearchConfigOverlay, finish_reason_from_stop_reason,
44};
45use roder_api::inference_routing::{InferenceRoutingOutcome, ModelSelectionMode};
46use roder_api::lifecycle::{
47    LifecycleMetricsSnapshot, TurnCleanupOwnership, TurnCleanupState, TurnLifecycleReason,
48    TurnLifecycleRecord, TurnLifecycleSnapshot, TurnLifecycleState, turn_lifecycle_snapshot,
49};
50use roder_api::policy_mode::{PolicyDecision, PolicyMode};
51use roder_api::reliability::{
52    ReliabilityContext, ReliabilityDetails, ReliabilityErrorClass, ReliabilityLimitDecision,
53    ReliabilityLimitRecorded, ReliabilityRequestPolicy, ReliabilityRetryDecision,
54    ReliabilityRetryRecorded, provider_retry_delay_ms,
55};
56use roder_api::remote_runner::{
57    RemoteRunnerProvider, RemoteRunnerSession, RemoteWorkspace, RunnerDestination,
58    RunnerSessionState, ThreadRunnerBinding,
59};
60use roder_api::subagents::SubagentDefinition;
61use roder_api::teams::{
62    TeamId, TeamMailboxMessage, TeamMailboxMessageKind, TeamMemberDescriptor, TeamMemberRole,
63    TeamMemberStatus,
64};
65use roder_api::thread::{
66    ThreadItemEvent, ThreadItemEventKind, ThreadMetadata, ThreadSnapshot, ThreadStore,
67    ThreadUsageMetadata, is_synthetic_event_thread_id, validate_thread_workspace,
68};
69use roder_api::tools::{ToolCall, ToolChoice, ToolExecutionContext, ToolRegistry, ToolResult};
70use roder_api::transcript::{
71    AssistantMessage, ErrorRecord, InputImage, ReasoningSummary, ToolCallRecord, ToolResultRecord,
72    TranscriptItem, UserMessage,
73};
74use roder_sandbox::ScopedFilesystem;
75use roder_sandbox::process::LocalProcessRunner;
76use roder_skills::{SkillRegistry, SkillRegistryOptions};
77use time::{Duration, OffsetDateTime};
78use tokio::sync::{Mutex, Notify, RwLock, oneshot};
79
80mod codex_v2;
81
82use crate::artifacts::{
83    ContextArtifactStore as FilesystemContextArtifactStore, default_context_artifact_dir,
84};
85use crate::bus::EventBus;
86use crate::dynamic_workflows::{
87    DynamicWorkflowEffortProfile, RuntimeDynamicWorkflowConfig, WorkflowTriggerDecision,
88    classify_workflow_trigger, ultracode_reasoning_level_for_model,
89};
90use crate::fake_provider::FakeInferenceEngine;
91use crate::goals::RuntimeGoalController;
92use crate::inference_routing::{
93    InferenceRoutingRequest, RuntimeInferenceRouterConfig, collect_inference_routing_candidates,
94    route_inference_selection, transcript_failure_count_since,
95};
96use crate::instructions::{
97    apply_agent_swarm_mode, apply_codex_multi_agent_mode, apply_model_instruction_overlay,
98    apply_parallel_web_tools, apply_plan_mode, apply_runtime_profile, apply_task_ledger_required,
99    apply_thread_developer_instructions, apply_turn_developer_context,
100};
101use crate::policy_gate::DefaultPolicyGate;
102use crate::reliability::{
103    ReliabilityLimitHit, RuntimeReliabilityConfig, TurnReliabilityState,
104    provider_stream_retry_cause,
105};
106pub use crate::speed_policy::RuntimeSpeedPolicyConfig;
107use crate::speed_policy::{SpeedPolicyState, reasoning_from_decision};
108use crate::subagent_traces::{RuntimeAgentSwarmProgressSink, RuntimeSubagentTraceSink};
109use crate::teams::{TeamManager, TeamMemberStartRequest, TeamStartRequest, TeamState};
110use crate::thread_item_cache::{ThreadItemCache, ThreadItemCacheEntry};
111use crate::verification_gate::VerificationGateState;
112
113const MAX_TOOL_ROUNDS_PER_TURN: usize = 1024;
114/// Injected when `continue_on_failure_limit` turns a consecutive-tool-failure
115/// limit into a continuation, or as the empty-tool-call persistence nudge, so
116/// the model keeps working instead of ending the turn early.
117const RELIABILITY_CONTINUATION_PROMPT: &str = "Verify the task is fully complete. If any part remains unfinished or unverified, keep working using the available tools — try an alternative approach if one is failing. Only stop once the solution is complete and verified; if it is already complete, restate your final answer.";
118/// Capacity of the runtime event broadcast ring buffer. The TUI drains this on
119/// its render loop, which during an active turn only wakes at the ~6 FPS status
120/// animation cadence (backend events do not wake the input poll), so up to
121/// ~166ms of events buffer between drains. A reasoning-high provider that runs
122/// its whole tool loop inside one turn (e.g. claude-code) streams a firehose of
123/// `InferenceEventReceived` deltas plus per-step tool/thinking events; the old
124/// 1024-slot buffer overflowed in those windows and silently dropped tool and
125/// thinking rows from the live view. Sized for generous headroom across bursts
126/// and brief render stalls.
127const EVENT_BUS_CAPACITY: usize = 16_384;
128
129/// Turn id reported to session-scoped local hooks. `SessionStart` and
130/// `SessionEnd` bracket the session rather than any one turn, but the hook
131/// payload carries a turn id, so they share this synthetic value.
132const SESSION_HOOK_TURN_ID: &str = "session";
133pub(crate) const FINAL_ANSWER_PHASE: &str = "final_answer";
134pub(crate) const TASK_LEDGER_TOOL_NAME: &str = "task_ledger.update";
135const TASK_LEDGER_COMPLETION_REMINDER_LIMIT: u8 = 2;
136const TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS: u64 = 180;
137const TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT: u8 = 1;
138pub(crate) const MIN_CHILD_DEADLINE_SECONDS: u64 = 2;
139const MODEL_PROFILE_TRACE_KIND: &str = "model_profile_segment";
140const MODEL_SWITCH_SUMMARY_PREFIX: &str = "Model switch summary:";
141const PROVIDER_CLEANUP_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5);
142
143#[derive(Clone, Copy, Debug, Eq, PartialEq)]
144enum InferenceTimeoutAction {
145    ScoreableCheckpoint,
146    Finalization,
147}
148
149#[derive(Debug, Clone)]
150pub struct RuntimeConfig {
151    pub default_provider: String,
152    pub default_model: String,
153    pub reasoning: Option<String>,
154    pub auto_compact_token_limit: Option<u32>,
155    pub file_backed_dynamic_context: bool,
156    pub hosted_web_search: HostedWebSearchConfig,
157    pub tool_search: ToolSearchConfig,
158    pub provider_tool_search: HashMap<String, ToolSearchConfigOverlay>,
159    pub model_tool_search: HashMap<String, ToolSearchConfigOverlay>,
160    pub model_edit_tools: HashMap<String, String>,
161    pub model_parallel_tool_calls: HashMap<String, bool>,
162    pub model_profiles: HashMap<String, ModelHarnessProfile>,
163    pub tool_allowlist: Vec<String>,
164    /// Seconds a host-executed external tool call may stay unresolved before it fails with a timeout error.
165    pub external_tool_timeout_seconds: u64,
166    pub command_shell: String,
167    pub workspace: Option<String>,
168    pub policy_mode: PolicyMode,
169    /// Whether agent-swarm mode is active (roadmap 104). When on, the runtime
170    /// injects the swarm reminder into each turn's developer instructions so any
171    /// client benefits, not just the TUI.
172    pub agent_swarm_mode: bool,
173    /// Whether ultra mode is active. When on, the runtime injects proactive
174    /// multi-agent delegation policy for any model (not only Codex Sol/Terra
175    /// Ultra effort). Selecting Ultra reasoning on Sol/Terra still enables
176    /// proactive multi-agent for that model without requiring this flag.
177    pub ultra_mode: bool,
178    pub runtime_profile: RuntimeProfile,
179    pub inference_router: RuntimeInferenceRouterConfig,
180    pub speed_policy: RuntimeSpeedPolicyConfig,
181    pub dynamic_workflows: RuntimeDynamicWorkflowConfig,
182    pub reliability: RuntimeReliabilityConfig,
183    /// Positive wall-clock limit for a turn in any runtime profile. `None` or zero is unbounded.
184    pub turn_deadline_seconds: Option<u64>,
185    pub remote_runner_destination: Option<RunnerDestination>,
186    pub team_data_dir: Option<PathBuf>,
187    pub roadmap_data_dir: Option<PathBuf>,
188    pub media_generation: crate::media_generation::RuntimeMediaGenerationConfig,
189    /// `[review]` settings: the review sub-turn and its publishers.
190    pub review: crate::review::RuntimeReviewConfig,
191}
192
193impl Default for RuntimeConfig {
194    fn default() -> Self {
195        Self {
196            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
197            default_model: "mock".to_string(),
198            reasoning: None,
199            auto_compact_token_limit: None,
200            file_backed_dynamic_context: true,
201            hosted_web_search: HostedWebSearchConfig::cached(),
202            tool_search: ToolSearchConfig::default(),
203            provider_tool_search: HashMap::new(),
204            model_tool_search: HashMap::new(),
205            model_edit_tools: HashMap::new(),
206            model_parallel_tool_calls: HashMap::new(),
207            model_profiles: HashMap::new(),
208            tool_allowlist: Vec::new(),
209            external_tool_timeout_seconds: DEFAULT_EXTERNAL_TOOL_TIMEOUT_SECONDS,
210            command_shell: roder_api::command_shell::default_command_shell(),
211            workspace: None,
212            policy_mode: PolicyMode::Default,
213            agent_swarm_mode: false,
214            ultra_mode: false,
215            runtime_profile: RuntimeProfile::Interactive,
216            inference_router: RuntimeInferenceRouterConfig::default(),
217            speed_policy: RuntimeSpeedPolicyConfig::default(),
218            dynamic_workflows: RuntimeDynamicWorkflowConfig::default(),
219            reliability: RuntimeReliabilityConfig::default(),
220            turn_deadline_seconds: None,
221            remote_runner_destination: None,
222            team_data_dir: None,
223            roadmap_data_dir: None,
224            media_generation: crate::media_generation::RuntimeMediaGenerationConfig::default(),
225            review: crate::review::RuntimeReviewConfig::default(),
226        }
227    }
228}
229
230#[derive(Debug, Clone)]
231pub struct StartTurnRequest {
232    pub thread_id: ThreadId,
233    pub message: String,
234    pub images: Vec<InputImage>,
235    pub provider_override: Option<String>,
236    pub model_override: Option<String>,
237    pub reasoning_override: Option<String>,
238    pub workspace: String,
239    pub instructions: InstructionBundle,
240    /**
241     * Per-turn developer-authority context for this turn's InstructionBundle.
242     * Applies to every inference round of the turn, is never written to
243     * thread state, and does not carry over to later turns.
244     */
245    pub developer_context: Option<String>,
246    pub task_ledger_required: bool,
247    /**
248     * Per-turn provider service tier (OpenAI `service_tier`, e.g.
249     * `"priority"` for Fast mode). Carried to every inference round of the
250     * turn as `RuntimeHints::service_tier`; `None` keeps the provider
251     * default. Providers without a tier concept ignore it.
252     */
253    pub service_tier_override: Option<String>,
254}
255
256#[derive(Debug, Clone)]
257pub struct CreateThreadRequest {
258    pub title: Option<String>,
259    pub workspace: String,
260    pub workspace_id: Option<String>,
261    pub root_id: Option<String>,
262    pub provider: Option<String>,
263    pub model: Option<String>,
264    pub selection_mode: Option<ModelSelectionMode>,
265    /// Per-thread tool filter applied on top of the runtime allowlist. Empty = no filtering.
266    pub tool_allowlist: Vec<String>,
267    /// Host-supplied instructions added to the developer slot of every turn's inference request.
268    pub developer_instructions: Option<String>,
269    /// Host-executed tool specs advertised to the model on every turn of this thread.
270    pub external_tools: Vec<roder_api::tools::ToolSpec>,
271    /// Explicit remote-runner binding for the thread's native coding tools.
272    pub runner: Option<ThreadRunnerSelection>,
273}
274
275/**
276 * Thread-level remote-runner selection. The destination config is persisted
277 * with the thread, so secrets must reach the provider through its
278 * environment, not this config.
279 */
280#[derive(Debug, Clone)]
281pub struct ThreadRunnerSelection {
282    pub provider_id: String,
283    pub config: serde_json::Value,
284    /// Absolute path on the runner used as the thread's coding-tool workspace root.
285    pub workspace: String,
286    /**
287     * Extra absolute runner paths file reads may resolve under, beyond
288     * `workspace`. Writes and the working directory stay confined to
289     * `workspace`.
290     */
291    pub read_roots: Vec<String>,
292}
293
294#[derive(Debug, Clone, PartialEq, Eq)]
295pub struct PendingPlanExit {
296    pub thread_id: ThreadId,
297    pub turn_id: TurnId,
298    pub request_id: String,
299    pub target_mode: PolicyMode,
300    pub plan_summary: Option<String>,
301    pub next_steps: Vec<String>,
302    pub requested_at: OffsetDateTime,
303    pub expires_at: Option<OffsetDateTime>,
304}
305
306pub(crate) struct PendingToolApproval {
307    pub(crate) thread_id: ThreadId,
308    pub(crate) turn_id: TurnId,
309    pub(crate) tool_id: String,
310    pub(crate) tool_name: String,
311    pub(crate) call: roder_api::tools::ToolCall,
312    pub(crate) tx: oneshot::Sender<bool>,
313}
314
315pub(crate) struct PendingUserInput {
316    pub(crate) thread_id: ThreadId,
317    pub(crate) turn_id: TurnId,
318    pub(crate) tx: oneshot::Sender<serde_json::Value>,
319}
320
321/// Host answer to an external tool call delivered via `tools/resolve`.
322#[derive(Debug, Clone, PartialEq, Eq)]
323pub struct ExternalToolResolution {
324    pub output: String,
325    pub is_error: bool,
326}
327
328pub(crate) struct PendingExternalToolCall {
329    pub(crate) thread_id: ThreadId,
330    pub(crate) turn_id: TurnId,
331    pub(crate) tool_id: String,
332    pub(crate) tool_name: String,
333    pub(crate) tx: oneshot::Sender<ExternalToolResolution>,
334}
335
336#[derive(Clone)]
337struct ActiveTurnHandle {
338    thread_id: ThreadId,
339    abort: AbortHandle,
340    steers: Arc<Mutex<Vec<QueuedTurnSteer>>>,
341    steer_changed: tokio::sync::watch::Sender<u64>,
342    drain: Arc<TurnDrainHandle>,
343}
344
345/// Tracks a task after it has been interrupted and removed from the interactive
346/// active-turn map. This lets users start a follow-up turn immediately while a
347/// bounded runtime shutdown can still await the original task's cleanup.
348struct TurnDrainHandle {
349    thread_id: ThreadId,
350    interrupt_requested: AtomicBool,
351    interrupt_reason: Mutex<Option<TurnLifecycleReason>>,
352    completed: AtomicBool,
353    completed_notify: Notify,
354}
355
356/// Result of a bounded runtime drain. `Clean` proves the runtime's own turn
357/// tasks have reached their cleanup paths; provider-specific child-process
358/// reaping remains an explicit provider capability.
359#[derive(Debug, Clone, PartialEq, Eq)]
360pub enum RuntimeDrainOutcome {
361    Clean {
362        interrupted_turn_ids: Vec<TurnId>,
363    },
364    DeadlineExceeded {
365        interrupted_turn_ids: Vec<TurnId>,
366        remaining_turn_ids: Vec<TurnId>,
367    },
368    /// At least one lifecycle transition initiated by this drain could not be
369    /// durably appended. The runtime still made its best cleanup attempt, but
370    /// callers must not claim a persisted terminal state as proof of recovery.
371    PersistenceFailed {
372        interrupted_turn_ids: Vec<TurnId>,
373        remaining_turn_ids: Vec<TurnId>,
374    },
375}
376
377#[derive(Clone)]
378struct QueuedTurnSteer {
379    message: UserMessage,
380    mailbox_ack: Option<MailboxDeliveryAck>,
381}
382
383#[derive(Clone)]
384struct MailboxDeliveryAck {
385    team_id: TeamId,
386    message_ids: Vec<String>,
387}
388
389#[derive(Clone)]
390struct InheritedTurnContext {
391    workspace: String,
392    instructions: InstructionBundle,
393    developer_context: Option<String>,
394}
395
396#[derive(Debug, Clone, Default, PartialEq, Eq)]
397pub struct ThreadActivity {
398    pub active_turn_id: Option<TurnId>,
399    pub active_flags: Vec<String>,
400}
401
402/// Per-thread settings persisted at thread creation and applied to every turn.
403#[derive(Debug, Clone, Default)]
404pub(crate) struct ThreadTurnOverrides {
405    pub(crate) tool_allowlist: Vec<String>,
406    pub(crate) developer_instructions: Option<String>,
407    pub(crate) external_tools: Vec<roder_api::tools::ToolSpec>,
408}
409
410#[derive(Debug, Clone, Copy, PartialEq, Eq)]
411pub(crate) enum TurnRunOutcome {
412    Completed,
413    Stopped,
414}
415
416impl PendingPlanExit {
417    pub fn new(
418        thread_id: ThreadId,
419        turn_id: TurnId,
420        request_id: String,
421        target_mode: PolicyMode,
422        plan_summary: Option<String>,
423        next_steps: Vec<String>,
424    ) -> Self {
425        let requested_at = OffsetDateTime::now_utc();
426        Self {
427            thread_id,
428            turn_id,
429            request_id,
430            target_mode,
431            plan_summary,
432            next_steps,
433            requested_at,
434            expires_at: Some(requested_at + default_plan_exit_timeout()),
435        }
436    }
437
438    pub fn is_expired(&self, now: OffsetDateTime) -> bool {
439        self.expires_at.is_some_and(|expires_at| now >= expires_at)
440    }
441}
442
443pub fn default_plan_exit_timeout() -> Duration {
444    Duration::minutes(10)
445}
446
447pub const DEFAULT_EXTERNAL_TOOL_TIMEOUT_SECONDS: u64 = 300;
448
449fn format_mailbox_messages(team: &TeamState, messages: &[TeamMailboxMessage]) -> String {
450    messages
451        .iter()
452        .map(|message| {
453            let recipient = team
454                .members
455                .iter()
456                .find(|member| member.id == message.to_member_id)
457                .map(canonical_team_member_path)
458                .unwrap_or_else(|| format!("/root/{}", message.to_member_id));
459            let sender = message
460                .from_member_id
461                .as_deref()
462                .and_then(|id| team.members.iter().find(|member| member.id == id))
463                .map(canonical_team_member_path)
464                .unwrap_or_else(|| "/root".to_string());
465            let message_type = match message.kind {
466                TeamMailboxMessageKind::Message => "MESSAGE",
467                TeamMailboxMessageKind::NewTask => "NEW_TASK",
468                TeamMailboxMessageKind::FinalAnswer => "FINAL_ANSWER",
469            };
470            format!(
471                "Message Type: {message_type}\nTask name: {recipient}\nSender: {sender}\nPayload:\n{}",
472                message.text
473            )
474        })
475        .collect::<Vec<_>>()
476        .join("\n\n")
477}
478
479fn canonical_team_member_path(member: &TeamMemberDescriptor) -> String {
480    if let Some(agent_path) = member
481        .agent_path
482        .as_deref()
483        .filter(|path| *path == "/root" || path.starts_with("/root/"))
484    {
485        return agent_path.to_string();
486    }
487    if member.role == TeamMemberRole::Lead {
488        return "/root".to_string();
489    }
490    member
491        .task_name
492        .as_deref()
493        .filter(|name| !name.trim().is_empty())
494        .map(|name| format!("/root/{}", name.trim().trim_matches('/')))
495        .unwrap_or_else(|| format!("/root/{}", member.id))
496}
497
498fn is_codex_v2_team(team: &TeamState) -> bool {
499    team.members.iter().any(|member| {
500        member
501            .model
502            .as_deref()
503            .is_some_and(|model| model_supports_reasoning_effort(model, REASONING_ULTRA))
504    })
505}
506
507fn runtime_local_team_view(
508    mut team: TeamState,
509    active_thread_ids: &std::collections::HashSet<ThreadId>,
510) -> TeamState {
511    for member in &mut team.members {
512        if member.status == TeamMemberStatus::Running
513            && !active_thread_ids.contains(&member.thread_id)
514        {
515            member.status = TeamMemberStatus::Interrupted;
516            member.current_turn_id = None;
517        }
518    }
519    team
520}
521
522type RunnerToolExecutionKey = (String, String);
523type RunnerToolExecutionMutex = Mutex<()>;
524type RunnerToolExecutionLockRegistry =
525    Mutex<HashMap<RunnerToolExecutionKey, Weak<RunnerToolExecutionMutex>>>;
526
527pub struct Runtime {
528    pub bus: EventBus,
529    pub registry: ExtensionRegistry,
530    config: RwLock<RuntimeConfig>,
531    pub(crate) execution_lease: Option<Arc<crate::execution_lease::RuntimeExecutionLease>>,
532    pending_plan_exit: RwLock<Option<PendingPlanExit>>,
533    pub(crate) pending_tool_approvals: Mutex<HashMap<String, PendingToolApproval>>,
534    pub(crate) pending_user_inputs: Mutex<HashMap<String, PendingUserInput>>,
535    pub(crate) pending_external_tool_calls: Mutex<HashMap<String, PendingExternalToolCall>>,
536    /// Serializes turn admission with shutdown quiescing. A drain takes this
537    /// gate before flipping `accepting_turns` and snapshotting active work so a
538    /// turn that observed the old flag cannot be inserted after the snapshot.
539    turn_admission: Mutex<()>,
540    active_turns: RwLock<HashMap<TurnId, ActiveTurnHandle>>,
541    turn_drains: RwLock<HashMap<TurnId, Arc<TurnDrainHandle>>>,
542    /// Provider cleanup handles register synchronously through the inference
543    /// tool-executor context. They are intentionally separate from active turn
544    /// admission so an interrupted turn can remain drainable after a follow-up
545    /// turn starts on the same thread.
546    provider_turn_cleanups: std::sync::Mutex<HashMap<TurnId, Arc<dyn ProviderTurnCleanup>>>,
547    accepting_turns: AtomicBool,
548    /// Monotonic counter used by bounded drains to surface lifecycle-state
549    /// persistence failures without making an individual provider turn fail.
550    lifecycle_persistence_failures: AtomicUsize,
551    lifecycle_shutdown_drains: AtomicUsize,
552    lifecycle_clean_shutdowns: AtomicUsize,
553    lifecycle_deadline_exceeded: AtomicUsize,
554    lifecycle_persistence_failed_drains: AtomicUsize,
555    lifecycle_restart_reconciliations: AtomicUsize,
556    lifecycle_shutdown_drain_duration_ms_total: AtomicUsize,
557    provider_cleanup_confirmed: AtomicUsize,
558    provider_cleanup_timed_out: AtomicUsize,
559    provider_cleanup_unknown: AtomicUsize,
560    active_turns_changed: Notify,
561    active_turn_selections: RwLock<HashMap<TurnId, ModelSelectionMode>>,
562    compaction_templates: RwLock<HashMap<ThreadId, AgentInferenceRequest>>,
563    thread_admission_gates: Mutex<HashMap<ThreadId, Arc<Mutex<()>>>>,
564    /// Threads that have already dispatched a `SessionStart` local hook.
565    ///
566    /// `ThreadCreated` fires once, but `ThreadLoaded` fires on every read of the
567    /// thread from its store, so without this a session ran its setup hooks
568    /// several times — including after the turn had already stopped.
569    session_hook_started: RwLock<HashSet<ThreadId>>,
570    active_turn_contexts: RwLock<HashMap<TurnId, InheritedTurnContext>>,
571    /// Process-local execution boundary. Local/CLI runtimes default to true;
572    /// hosted runtime pools set this false so missing runner metadata fails
573    /// closed instead of exposing the host workspace.
574    allow_local_workspaces: AtomicBool,
575    // Spawn-time live authority retained for every reusable turn of a long-lived teammate.
576    // This stays process-local because developer_context is explicitly volatile and must never
577    // be persisted to thread state.
578    team_member_turn_contexts: Mutex<HashMap<ThreadId, InheritedTurnContext>>,
579    workspace: PathBuf,
580    pub(crate) teams: TeamManager,
581    agent_team_spawn_lock: Mutex<()>,
582    pub(crate) roadmaps: Mutex<roder_roadmap::RoadmapRuntime>,
583    /// Completed reviews, most recent last, so a later publish can resolve a
584    /// review id back to its findings. Bounded; see `review::run`.
585    pub(crate) reviews: Mutex<Vec<crate::review::ReviewRecord>>,
586    pub(crate) goals: Arc<RuntimeGoalController>,
587    context_artifacts: roder_api::artifacts::ContextArtifactStore,
588    pub(crate) thread_store: Option<Arc<dyn ThreadStore>>,
589    thread_item_cache: Mutex<ThreadItemCache>,
590    pub(crate) tool_registry: ToolRegistry,
591    media_generation: Arc<crate::media_generation::MediaGenerationService>,
592    pub(crate) skills: RwLock<SkillRegistry>,
593    /// Lazily-started bounded dispatch of emitted events to registry
594    /// `EventSink`s (process extensions etc.); see `event_sink_dispatch`.
595    event_sink_dispatcher: tokio::sync::OnceCell<crate::event_sink_dispatch::EventSinkDispatcher>,
596    pub(crate) compaction_hysteresis:
597        std::sync::Mutex<HashMap<ThreadId, crate::compaction_runtime::CompactionState>>,
598    /// Per-thread agent-swarm mode overrides (roadmap 104). A thread present in
599    /// this map uses its stored value; absent threads fall back to the
600    /// runtime-global `RuntimeConfig.agent_swarm_mode` default. This mirrors the
601    /// team per-member policy-mode override idiom so swarm mode is per-thread
602    /// like the other `thread/*` operations, without a separate runtime.
603    agent_swarm_modes: RwLock<HashMap<ThreadId, bool>>,
604    /// Per-thread ultra mode overrides. A thread present in this map uses its
605    /// stored value; absent threads fall back to the runtime-global
606    /// `RuntimeConfig.ultra_mode` default.
607    ultra_modes: RwLock<HashMap<ThreadId, bool>>,
608    /**
609     * Live remote-runner sessions keyed by thread. Each per-thread mutex is
610     * held across provider initialization, making the first workspace-tool
611     * access a singleflight while allowing unrelated threads to initialize in
612     * parallel. Failed initialization is shared with current waiters, then
613     * evicted so a later tool call can retry.
614     */
615    runner_sessions: Arc<Mutex<HashMap<ThreadId, Arc<RunnerSessionSlot>>>>,
616    /**
617     * Outer tool-call locks keyed by the actual remote session, not the
618     * per-thread destination id. Hosted runtimes are tenant-scoped, so this
619     * serializes threads that share one sandbox without coupling different
620     * tenants or independent runner sessions.
621     */
622    runner_tool_execution_locks: RunnerToolExecutionLockRegistry,
623}
624
625#[derive(Clone)]
626struct CachedRunnerSession {
627    destination: RunnerDestination,
628    session: Arc<dyn RemoteRunnerSession>,
629}
630
631struct RunnerSessionSlot {
632    provider_id: String,
633    destination_id: String,
634    state: Mutex<RunnerSessionSlotState>,
635    initialized: Notify,
636}
637
638enum RunnerSessionSlotState {
639    Empty,
640    Initializing,
641    Ready(CachedRunnerSession),
642    Failed(Arc<str>),
643}
644
645impl RunnerSessionSlot {
646    fn empty(destination: &RunnerDestination) -> Self {
647        Self {
648            provider_id: destination.provider_id.clone(),
649            destination_id: destination.id.clone(),
650            state: Mutex::new(RunnerSessionSlotState::Empty),
651            initialized: Notify::new(),
652        }
653    }
654
655    fn ready(session: CachedRunnerSession) -> Self {
656        Self {
657            provider_id: session.destination.provider_id.clone(),
658            destination_id: session.destination.id.clone(),
659            state: Mutex::new(RunnerSessionSlotState::Ready(session)),
660            initialized: Notify::new(),
661        }
662    }
663
664    fn matches(&self, destination: &RunnerDestination) -> bool {
665        self.provider_id == destination.provider_id && self.destination_id == destination.id
666    }
667}
668
669impl Runtime {
670    pub fn new(registry: ExtensionRegistry, config: RuntimeConfig) -> anyhow::Result<Self> {
671        if registry.inference_engines.is_empty() {
672            anyhow::bail!("at least one inference engine must be registered");
673        }
674        validate_runtime_config_reasoning(&config)?;
675        validate_runtime_inference_router_config(&registry, &config)?;
676
677        let bus = EventBus::new(EVENT_BUS_CAPACITY);
678        let thread_store = registry
679            .thread_stores
680            .first()
681            .map(|factory| factory.create());
682        let mut tool_registry = ToolRegistry::default();
683        for contributor in &registry.tools {
684            contributor
685                .contribute(&mut tool_registry)
686                .with_context(|| format!("tool contributor {} failed", contributor.id()))?;
687        }
688        crate::agent_control_tools::contribute_agent_control_tools(&mut tool_registry)?;
689
690        let media_generation = Arc::new(crate::media_generation::MediaGenerationService::new(
691            registry.media_generator_providers.clone(),
692            config.media_generation.clone(),
693        ));
694        tool_registry.replace(Arc::new(
695            crate::media_generation::MediaGenerateImageTool::new(media_generation.clone()),
696        ));
697
698        let team_data_dir = config.team_data_dir.clone();
699        let workspace = config
700            .workspace
701            .clone()
702            .map(PathBuf::from)
703            .unwrap_or(std::env::current_dir()?);
704        let roadmap_data_dir = config
705            .roadmap_data_dir
706            .clone()
707            .unwrap_or_else(|| workspace.join(".roder"));
708        let context_artifacts = thread_store
709            .as_ref()
710            .and_then(|store| store.context_artifact_store())
711            .or_else(|| {
712                thread_store
713                    .as_ref()
714                    .and_then(|store| store.local_thread_root())
715                    .map(FilesystemContextArtifactStore::shared_thread_scoped)
716            })
717            .unwrap_or_else(|| {
718                FilesystemContextArtifactStore::shared_legacy(default_context_artifact_dir())
719            });
720        let goals = Arc::new(RuntimeGoalController::new(
721            bus.clone(),
722            thread_store.clone(),
723        ));
724        let runtime = Self {
725            bus,
726            registry,
727            config: RwLock::new(config),
728            execution_lease: None,
729            pending_plan_exit: RwLock::new(None),
730            pending_tool_approvals: Mutex::new(HashMap::new()),
731            pending_user_inputs: Mutex::new(HashMap::new()),
732            pending_external_tool_calls: Mutex::new(HashMap::new()),
733            turn_admission: Mutex::new(()),
734            active_turns: RwLock::new(HashMap::new()),
735            turn_drains: RwLock::new(HashMap::new()),
736            provider_turn_cleanups: std::sync::Mutex::new(HashMap::new()),
737            accepting_turns: AtomicBool::new(true),
738            lifecycle_persistence_failures: AtomicUsize::new(0),
739            lifecycle_shutdown_drains: AtomicUsize::new(0),
740            lifecycle_clean_shutdowns: AtomicUsize::new(0),
741            lifecycle_deadline_exceeded: AtomicUsize::new(0),
742            lifecycle_persistence_failed_drains: AtomicUsize::new(0),
743            lifecycle_restart_reconciliations: AtomicUsize::new(0),
744            lifecycle_shutdown_drain_duration_ms_total: AtomicUsize::new(0),
745            provider_cleanup_confirmed: AtomicUsize::new(0),
746            provider_cleanup_timed_out: AtomicUsize::new(0),
747            provider_cleanup_unknown: AtomicUsize::new(0),
748            active_turns_changed: Notify::new(),
749            active_turn_selections: RwLock::new(HashMap::new()),
750            compaction_templates: RwLock::new(HashMap::new()),
751            thread_admission_gates: Mutex::new(HashMap::new()),
752            session_hook_started: RwLock::new(HashSet::new()),
753            active_turn_contexts: RwLock::new(HashMap::new()),
754            allow_local_workspaces: AtomicBool::new(true),
755            team_member_turn_contexts: Mutex::new(HashMap::new()),
756            workspace: workspace.clone(),
757            teams: TeamManager::new(
758                team_data_dir.unwrap_or_else(crate::teams::default_team_data_dir),
759            ),
760            agent_team_spawn_lock: Mutex::new(()),
761            roadmaps: Mutex::new(roder_roadmap::RoadmapRuntime::new(
762                workspace,
763                roadmap_data_dir,
764            )),
765            reviews: Mutex::new(Vec::new()),
766            goals,
767            context_artifacts,
768            thread_store,
769            thread_item_cache: Mutex::new(ThreadItemCache::default()),
770            tool_registry,
771            media_generation,
772            skills: RwLock::new(SkillRegistry::load(SkillRegistryOptions::new(
773                PathBuf::new(),
774            ))),
775            event_sink_dispatcher: tokio::sync::OnceCell::new(),
776            compaction_hysteresis: crate::compaction_runtime::compaction_hysteresis_state(),
777            agent_swarm_modes: RwLock::new(HashMap::new()),
778            ultra_modes: RwLock::new(HashMap::new()),
779            runner_sessions: Arc::new(Mutex::new(HashMap::new())),
780            runner_tool_execution_locks: Mutex::new(HashMap::new()),
781        };
782        runtime.bus.emit(RoderEvent::RuntimeStarted(RuntimeStarted {
783            timestamp: OffsetDateTime::now_utc(),
784        }));
785        for manifest in &runtime.registry.manifests {
786            runtime
787                .bus
788                .emit(RoderEvent::ExtensionRegistered(ExtensionRegistered {
789                    extension_id: manifest.id.clone(),
790                    timestamp: OffsetDateTime::now_utc(),
791                }));
792        }
793        Ok(runtime)
794    }
795
796    pub fn from_engine(engine: Arc<dyn InferenceEngine>) -> anyhow::Result<Self> {
797        let mut builder = roder_api::extension::ExtensionRegistryBuilder::new();
798        builder.inference_engine(engine);
799        Self::new(builder.build()?, RuntimeConfig::default())
800    }
801
802    pub fn fake() -> anyhow::Result<Self> {
803        Self::from_engine(Arc::new(FakeInferenceEngine))
804    }
805
806    pub fn subscribe_events(&self) -> tokio::sync::broadcast::Receiver<EventEnvelope> {
807        self.bus.subscribe()
808    }
809
810    /// Returns process-local, redacted lifecycle health counters. The snapshot
811    /// intentionally has fixed fields and contains no provider, command,
812    /// process, thread, or turn identifiers.
813    pub fn lifecycle_metrics(&self) -> LifecycleMetricsSnapshot {
814        LifecycleMetricsSnapshot {
815            shutdown_drain_count: self.lifecycle_shutdown_drains.load(Ordering::Acquire) as u64,
816            clean_shutdown_count: self.lifecycle_clean_shutdowns.load(Ordering::Acquire) as u64,
817            deadline_exceeded_count: self.lifecycle_deadline_exceeded.load(Ordering::Acquire)
818                as u64,
819            persistence_failed_count: self
820                .lifecycle_persistence_failed_drains
821                .load(Ordering::Acquire) as u64,
822            restart_reconciliation_count: self
823                .lifecycle_restart_reconciliations
824                .load(Ordering::Acquire) as u64,
825            lifecycle_persistence_failure_count: self
826                .lifecycle_persistence_failures
827                .load(Ordering::Acquire) as u64,
828            shutdown_drain_duration_ms_total: self
829                .lifecycle_shutdown_drain_duration_ms_total
830                .load(Ordering::Acquire) as u64,
831            provider_cleanup_confirmed_count: self
832                .provider_cleanup_confirmed
833                .load(Ordering::Acquire) as u64,
834            provider_cleanup_timed_out_count: self
835                .provider_cleanup_timed_out
836                .load(Ordering::Acquire) as u64,
837            provider_cleanup_unknown_count: self.provider_cleanup_unknown.load(Ordering::Acquire)
838                as u64,
839        }
840    }
841
842    async fn persist_turn_lifecycle_record(&self, record: &TurnLifecycleRecord) -> bool {
843        let Some(store) = &self.thread_store else {
844            return true;
845        };
846        let state = match record.extension_state() {
847            Ok(state) => state,
848            Err(_) => {
849                self.lifecycle_persistence_failures
850                    .fetch_add(1, Ordering::AcqRel);
851                return false;
852            }
853        };
854        // Lifecycle diagnostics must not turn a provider result into a failed
855        // turn merely because a third-party store lacks extension-state support.
856        // JSONL/Postgres stores persist this append-only record atomically at
857        // their own storage boundary.
858        if store
859            .append_extension_state(&record.thread_id, &state)
860            .await
861            .is_err()
862        {
863            self.lifecycle_persistence_failures
864                .fetch_add(1, Ordering::AcqRel);
865            return false;
866        }
867        true
868    }
869
870    async fn record_turn_lifecycle(
871        &self,
872        thread_id: ThreadId,
873        turn_id: TurnId,
874        state: TurnLifecycleState,
875        cleanup: TurnCleanupState,
876        reason: Option<TurnLifecycleReason>,
877    ) -> TurnLifecycleRecord {
878        self.record_turn_lifecycle_with_ownership(
879            thread_id,
880            turn_id,
881            state,
882            cleanup,
883            reason,
884            TurnCleanupOwnership::RuntimeTaskOnly,
885        )
886        .await
887    }
888
889    async fn record_turn_lifecycle_with_ownership(
890        &self,
891        thread_id: ThreadId,
892        turn_id: TurnId,
893        state: TurnLifecycleState,
894        cleanup: TurnCleanupState,
895        reason: Option<TurnLifecycleReason>,
896        ownership: TurnCleanupOwnership,
897    ) -> TurnLifecycleRecord {
898        let record = TurnLifecycleRecord::new(
899            thread_id,
900            turn_id,
901            state,
902            cleanup,
903            reason,
904            OffsetDateTime::now_utc(),
905        )
906        .with_ownership(ownership);
907        let _ = self.persist_turn_lifecycle_record(&record).await;
908        self.emit(RoderEvent::TurnLifecycleUpdated(record.clone()))
909            .await;
910        record
911    }
912
913    pub(crate) fn register_provider_turn_cleanup(
914        &self,
915        turn_id: &TurnId,
916        cleanup: Arc<dyn ProviderTurnCleanup>,
917    ) {
918        if let Ok(mut cleanups) = self.provider_turn_cleanups.lock() {
919            cleanups.insert(turn_id.clone(), cleanup);
920        }
921    }
922
923    fn provider_cleanup_ownership(&self, turn_id: &TurnId) -> TurnCleanupOwnership {
924        self.provider_turn_cleanups
925            .lock()
926            .ok()
927            .and_then(|cleanups| cleanups.get(turn_id).cloned())
928            .map(|cleanup| cleanup.ownership())
929            .unwrap_or(TurnCleanupOwnership::RuntimeTaskOnly)
930    }
931
932    async fn await_provider_turn_cleanup(
933        &self,
934        turn_id: &TurnId,
935    ) -> (TurnCleanupState, TurnCleanupOwnership) {
936        let cleanup = self
937            .provider_turn_cleanups
938            .lock()
939            .ok()
940            .and_then(|mut cleanups| cleanups.remove(turn_id));
941        let Some(cleanup) = cleanup else {
942            return (
943                TurnCleanupState::Unknown,
944                TurnCleanupOwnership::RuntimeTaskOnly,
945            );
946        };
947        let ownership = cleanup.ownership();
948        match tokio::time::timeout(PROVIDER_CLEANUP_TIMEOUT, cleanup.wait_for_cleanup()).await {
949            Ok(Ok(())) => {
950                self.provider_cleanup_confirmed
951                    .fetch_add(1, Ordering::AcqRel);
952                (
953                    TurnCleanupState::Completed,
954                    TurnCleanupOwnership::ProviderCleanupConfirmed,
955                )
956            }
957            Ok(Err(_)) => {
958                self.provider_cleanup_unknown.fetch_add(1, Ordering::AcqRel);
959                (TurnCleanupState::Unknown, ownership)
960            }
961            Err(_) => {
962                self.provider_cleanup_timed_out
963                    .fetch_add(1, Ordering::AcqRel);
964                (TurnCleanupState::TimedOut, ownership)
965            }
966        }
967    }
968
969    fn record_lifecycle_drain_outcome(
970        &self,
971        started_at: tokio::time::Instant,
972        outcome: &RuntimeDrainOutcome,
973    ) {
974        self.lifecycle_shutdown_drains
975            .fetch_add(1, Ordering::AcqRel);
976        self.lifecycle_shutdown_drain_duration_ms_total.fetch_add(
977            started_at.elapsed().as_millis().min(usize::MAX as u128) as usize,
978            Ordering::AcqRel,
979        );
980        match outcome {
981            RuntimeDrainOutcome::Clean { .. } => {
982                self.lifecycle_clean_shutdowns
983                    .fetch_add(1, Ordering::AcqRel);
984            }
985            RuntimeDrainOutcome::DeadlineExceeded { .. } => {
986                self.lifecycle_deadline_exceeded
987                    .fetch_add(1, Ordering::AcqRel);
988            }
989            RuntimeDrainOutcome::PersistenceFailed { .. } => {
990                self.lifecycle_persistence_failed_drains
991                    .fetch_add(1, Ordering::AcqRel);
992            }
993        }
994    }
995
996    fn lifecycle_record_for_event(event: &RoderEvent) -> Option<TurnLifecycleRecord> {
997        match event {
998            RoderEvent::TurnStarted(event) => Some(TurnLifecycleRecord::new(
999                event.thread_id.clone(),
1000                event.turn_id.clone(),
1001                TurnLifecycleState::Running,
1002                TurnCleanupState::NotRequested,
1003                None,
1004                event.timestamp,
1005            )),
1006            // `run_turn` persists completed turns after it has awaited a
1007            // registered provider cleanup handle. Do not let the generic event
1008            // projection overwrite that acknowledgement with an unproven state.
1009            RoderEvent::TurnCompleted(_) => None,
1010            // Some failure paths already have an event before the turn wrapper
1011            // reaches its cleanup acknowledgement. Preserve a durable fallback
1012            // reason here; the wrapper appends a later record with the more
1013            // precise cleanup outcome when a provider registered one.
1014            RoderEvent::TurnFailed(event) => Some(TurnLifecycleRecord::new(
1015                event.thread_id.clone(),
1016                event.turn_id.clone(),
1017                TurnLifecycleState::Failed,
1018                TurnCleanupState::Unknown,
1019                Some(if event.error_kind.as_deref() == Some("deadline_timeout") {
1020                    TurnLifecycleReason::DeadlineExceeded
1021                } else {
1022                    TurnLifecycleReason::ProviderFailure
1023                }),
1024                event.timestamp,
1025            )),
1026            RoderEvent::TurnDeadlineExceeded(_) | RoderEvent::TurnInterrupted(_) => None,
1027            RoderEvent::TurnLifecycleUpdated(_) => None,
1028            _ => None,
1029        }
1030    }
1031
1032    pub fn registry(&self) -> &ExtensionRegistry {
1033        &self.registry
1034    }
1035
1036    pub fn media_generation(&self) -> Arc<crate::media_generation::MediaGenerationService> {
1037        self.media_generation.clone()
1038    }
1039
1040    pub fn context_artifacts(&self) -> roder_api::artifacts::ContextArtifactStore {
1041        self.context_artifacts.clone()
1042    }
1043
1044    pub async fn execute_workflow_tool(
1045        &self,
1046        thread_id: ThreadId,
1047        tool_name: &str,
1048        arguments: serde_json::Value,
1049    ) -> anyhow::Result<ToolResult> {
1050        let Some(executor) = self.tool_registry.get(tool_name) else {
1051            anyhow::bail!("tool not found: {tool_name}");
1052        };
1053        let tool_call = ToolCall {
1054            id: format!("slash-{tool_name}"),
1055            name: tool_name.to_string(),
1056            raw_arguments: serde_json::to_string(&arguments)?,
1057            arguments,
1058            thread_id: thread_id.clone(),
1059            turn_id: "slash-command".to_string(),
1060        };
1061        let runtime_config = self.status().await;
1062        let ctx = self.tool_execution_context(
1063            thread_id,
1064            "slash-command".to_string(),
1065            runtime_config.policy_mode,
1066            runtime_config.workspace.as_deref(),
1067            Some(&runtime_config.command_shell),
1068        );
1069        executor.execute(ctx, tool_call).await
1070    }
1071
1072    pub(crate) fn tool_execution_context(
1073        &self,
1074        thread_id: ThreadId,
1075        turn_id: TurnId,
1076        mode: PolicyMode,
1077        workspace: Option<&str>,
1078        command_shell: Option<&str>,
1079    ) -> ToolExecutionContext {
1080        let mut ctx = ToolExecutionContext::new(thread_id, turn_id, mode)
1081            .with_command_shell(command_shell.unwrap_or_default())
1082            .with_process_runner(Arc::new(LocalProcessRunner))
1083            .with_context_artifacts(self.context_artifacts.backend())
1084            .with_goal_controller(self.goals.clone())
1085            .with_subagent_trace_sink(Arc::new(RuntimeSubagentTraceSink::new(
1086                self.bus.clone(),
1087                self.thread_store.clone(),
1088            )))
1089            .with_swarm_progress_sink(Arc::new(RuntimeAgentSwarmProgressSink::new(
1090                self.bus.clone(),
1091                self.thread_store.clone(),
1092            )));
1093        if let Some(workspace) = workspace {
1094            ctx = ctx.with_workspace_handle(Arc::new(ScopedFilesystem::new(workspace)));
1095        }
1096        ctx
1097    }
1098
1099    pub async fn status(&self) -> RuntimeConfig {
1100        self.config.read().await.clone()
1101    }
1102
1103    pub async fn set_skills(&self, skills: SkillRegistry) {
1104        *self.skills.write().await = skills;
1105    }
1106
1107    pub async fn skills_snapshot(&self) -> SkillRegistry {
1108        self.skills.read().await.clone()
1109    }
1110
1111    pub fn workspace(&self) -> PathBuf {
1112        self.workspace.clone()
1113    }
1114
1115    /// Controls whether native workspace tools may execute without an
1116    /// explicit remote-runner binding. Hosted runtimes disable this after
1117    /// construction; ordinary local runtimes retain the compatible default.
1118    pub fn set_allow_local_workspaces(&self, allowed: bool) {
1119        self.allow_local_workspaces
1120            .store(allowed, Ordering::Relaxed);
1121    }
1122
1123    pub fn allows_local_workspaces(&self) -> bool {
1124        self.allow_local_workspaces.load(Ordering::Relaxed)
1125    }
1126
1127    pub async fn set_remote_runner_destination(&self, destination: Option<RunnerDestination>) {
1128        let lifecycle = destination.as_ref().map(|destination| RunnerLifecycle {
1129            destination_id: destination.id.clone(),
1130            provider_id: destination.provider_id.clone(),
1131            state: "configured".to_string(),
1132            session_id: None,
1133            timestamp: OffsetDateTime::now_utc(),
1134        });
1135        self.config.write().await.remote_runner_destination = destination;
1136        if let Some(lifecycle) = lifecycle {
1137            self.emit(RoderEvent::RunnerLifecycle(lifecycle)).await;
1138        } else {
1139            self.emit(RoderEvent::RunnerLifecycle(RunnerLifecycle {
1140                destination_id: "local".to_string(),
1141                provider_id: "local".to_string(),
1142                state: "local_fallback".to_string(),
1143                session_id: None,
1144                timestamp: OffsetDateTime::now_utc(),
1145            }))
1146            .await;
1147        }
1148    }
1149
1150    pub async fn set_file_backed_dynamic_context(&self, enabled: bool) -> RuntimeConfig {
1151        let mut cfg = self.config.write().await;
1152        cfg.file_backed_dynamic_context = enabled;
1153        cfg.clone()
1154    }
1155
1156    pub async fn set_command_shell(&self, shell: String) -> RuntimeConfig {
1157        let mut cfg = self.config.write().await;
1158        cfg.command_shell = shell;
1159        cfg.clone()
1160    }
1161
1162    pub async fn pending_plan_exit(&self) -> Option<PendingPlanExit> {
1163        let mut pending = self.pending_plan_exit.write().await;
1164        let current = pending.clone()?;
1165        if !current.is_expired(OffsetDateTime::now_utc()) {
1166            return Some(current);
1167        }
1168        *pending = None;
1169        drop(pending);
1170        self.emit_plan_exit_resolved(&current, false, self.status().await.policy_mode)
1171            .await;
1172        None
1173    }
1174
1175    pub async fn set_policy_mode(
1176        &self,
1177        mode: PolicyMode,
1178        reason: Option<String>,
1179    ) -> anyhow::Result<RuntimeConfig> {
1180        let mut cfg = self.config.write().await;
1181        let previous_mode = cfg.policy_mode;
1182        cfg.policy_mode = mode;
1183        let next = cfg.clone();
1184        drop(cfg);
1185        self.emit(RoderEvent::PolicyModeChanged(PolicyModeChanged {
1186            thread_id: "runtime".to_string(),
1187            turn_id: None,
1188            previous_mode,
1189            new_mode: mode,
1190            reason,
1191            timestamp: OffsetDateTime::now_utc(),
1192        }))
1193        .await;
1194        self.auto_resolve_pending_tool_approvals_for_mode(mode)
1195            .await;
1196        Ok(next)
1197    }
1198
1199    /// Toggle agent-swarm mode for the runtime (roadmap 104). When enabled, the
1200    /// swarm reminder is injected into each turn's developer instructions so
1201    /// every app-server/SDK client gets it, not only the TUI.
1202    pub async fn set_agent_swarm_mode(
1203        &self,
1204        enabled: bool,
1205        trigger: roder_api::subagents::AgentSwarmModeTrigger,
1206    ) -> anyhow::Result<RuntimeConfig> {
1207        let mut cfg = self.config.write().await;
1208        cfg.agent_swarm_mode = enabled;
1209        let next = cfg.clone();
1210        drop(cfg);
1211        self.emit(RoderEvent::AgentSwarmModeChanged(
1212            roder_api::subagents::AgentSwarmModeChanged {
1213                thread_id: "runtime".to_string(),
1214                turn_id: None,
1215                enabled,
1216                trigger,
1217                timestamp: OffsetDateTime::now_utc(),
1218            },
1219        ))
1220        .await;
1221        Ok(next)
1222    }
1223
1224    /// Toggle agent-swarm mode for a single thread (roadmap 104). The override is
1225    /// stored per thread so toggling swarm mode on one thread does not leak the
1226    /// reminder into other threads sharing the runtime; absent threads fall back
1227    /// to the runtime-global default. The emitted `AgentSwarmModeChanged` event
1228    /// carries the real `thread_id`.
1229    pub async fn set_agent_swarm_mode_for_thread(
1230        &self,
1231        thread_id: &str,
1232        enabled: bool,
1233        trigger: roder_api::subagents::AgentSwarmModeTrigger,
1234    ) -> bool {
1235        {
1236            let mut modes = self.agent_swarm_modes.write().await;
1237            modes.insert(thread_id.to_string(), enabled);
1238        }
1239        self.emit(RoderEvent::AgentSwarmModeChanged(
1240            roder_api::subagents::AgentSwarmModeChanged {
1241                thread_id: thread_id.to_string(),
1242                turn_id: None,
1243                enabled,
1244                trigger,
1245                timestamp: OffsetDateTime::now_utc(),
1246            },
1247        ))
1248        .await;
1249        enabled
1250    }
1251
1252    /// Resolve whether agent-swarm mode is active for `thread_id`: the per-thread
1253    /// override when present, otherwise the runtime-global default. The turn loop
1254    /// uses this to decide whether to inject the swarm reminder.
1255    pub async fn effective_agent_swarm_mode_for_thread(&self, thread_id: &str) -> bool {
1256        if let Some(enabled) = self.agent_swarm_modes.read().await.get(thread_id).copied() {
1257            return enabled;
1258        }
1259        self.status().await.agent_swarm_mode
1260    }
1261
1262    /// Toggle ultra mode for the runtime. When enabled, proactive multi-agent
1263    /// policy is injected into each turn's developer instructions for any
1264    /// model, so every app-server/SDK client gets it (not only the TUI).
1265    pub async fn set_ultra_mode(
1266        &self,
1267        enabled: bool,
1268        trigger: roder_api::subagents::UltraModeTrigger,
1269    ) -> anyhow::Result<RuntimeConfig> {
1270        let mut cfg = self.config.write().await;
1271        cfg.ultra_mode = enabled;
1272        let next = cfg.clone();
1273        drop(cfg);
1274        self.emit(RoderEvent::UltraModeChanged(
1275            roder_api::subagents::UltraModeChanged {
1276                thread_id: "runtime".to_string(),
1277                turn_id: None,
1278                enabled,
1279                trigger,
1280                timestamp: OffsetDateTime::now_utc(),
1281            },
1282        ))
1283        .await;
1284        Ok(next)
1285    }
1286
1287    /// Toggle ultra mode for a single thread. The override is stored per thread
1288    /// so toggling on one thread does not leak proactive multi-agent policy into
1289    /// other threads sharing the runtime; absent threads fall back to the
1290    /// runtime-global default.
1291    pub async fn set_ultra_mode_for_thread(
1292        &self,
1293        thread_id: &str,
1294        enabled: bool,
1295        trigger: roder_api::subagents::UltraModeTrigger,
1296    ) -> bool {
1297        {
1298            let mut modes = self.ultra_modes.write().await;
1299            modes.insert(thread_id.to_string(), enabled);
1300        }
1301        self.emit(RoderEvent::UltraModeChanged(
1302            roder_api::subagents::UltraModeChanged {
1303                thread_id: thread_id.to_string(),
1304                turn_id: None,
1305                enabled,
1306                trigger,
1307                timestamp: OffsetDateTime::now_utc(),
1308            },
1309        ))
1310        .await;
1311        enabled
1312    }
1313
1314    /// Resolve whether ultra mode is active for `thread_id`: the per-thread
1315    /// override when present, otherwise the runtime-global default. The turn
1316    /// loop uses this (together with Sol/Terra Ultra effort) to decide whether
1317    /// to inject proactive multi-agent policy.
1318    pub async fn effective_ultra_mode_for_thread(&self, thread_id: &str) -> bool {
1319        if let Some(enabled) = self.ultra_modes.read().await.get(thread_id).copied() {
1320            return enabled;
1321        }
1322        self.status().await.ultra_mode
1323    }
1324
1325    pub async fn set_hosted_web_search(
1326        &self,
1327        mode: HostedWebSearchMode,
1328    ) -> anyhow::Result<RuntimeConfig> {
1329        let mut cfg = self.config.write().await;
1330        cfg.hosted_web_search = HostedWebSearchConfig { mode };
1331        Ok(cfg.clone())
1332    }
1333
1334    async fn auto_resolve_pending_tool_approvals_for_mode(&self, mode: PolicyMode) {
1335        let gate = DefaultPolicyGate::new();
1336        let mut pending = self.pending_tool_approvals.lock().await;
1337        let approval_ids = pending
1338            .iter()
1339            .filter_map(|(approval_id, approval)| {
1340                let ctx = ToolExecutionContext::new(
1341                    approval.thread_id.clone(),
1342                    approval.turn_id.clone(),
1343                    mode,
1344                );
1345                matches!(
1346                    gate.decide(&approval.call, mode, &ctx),
1347                    PolicyDecision::AutoApproved { .. }
1348                )
1349                .then_some(approval_id.clone())
1350            })
1351            .collect::<Vec<_>>();
1352        let approvals = approval_ids
1353            .into_iter()
1354            .filter_map(|approval_id| {
1355                pending
1356                    .remove(&approval_id)
1357                    .map(|approval| (approval_id, approval))
1358            })
1359            .collect::<Vec<_>>();
1360        drop(pending);
1361
1362        for (approval_id, approval) in approvals {
1363            let ctx = ToolExecutionContext::new(
1364                approval.thread_id.clone(),
1365                approval.turn_id.clone(),
1366                mode,
1367            );
1368            let decision = gate.decide(&approval.call, mode, &ctx);
1369            self.emit(RoderEvent::PolicyDecisionRecorded(PolicyDecisionRecorded {
1370                thread_id: approval.thread_id.clone(),
1371                turn_id: approval.turn_id.clone(),
1372                tool_id: approval.tool_id.clone(),
1373                tool_name: approval.tool_name.clone(),
1374                mode,
1375                decision,
1376                timestamp: OffsetDateTime::now_utc(),
1377            }))
1378            .await;
1379            if mode == PolicyMode::Bypass {
1380                self.emit(RoderEvent::PolicyBypassActive(PolicyBypassActive {
1381                    thread_id: approval.thread_id.clone(),
1382                    turn_id: approval.turn_id.clone(),
1383                    tool_id: approval.tool_id.clone(),
1384                    tool_name: approval.tool_name.clone(),
1385                    timestamp: OffsetDateTime::now_utc(),
1386                }))
1387                .await;
1388            }
1389            self.emit(RoderEvent::ApprovalResolved(ApprovalResolved {
1390                thread_id: approval.thread_id,
1391                turn_id: approval.turn_id,
1392                approval_id,
1393                tool_id: approval.tool_id,
1394                tool_name: approval.tool_name,
1395                approved: true,
1396                timestamp: OffsetDateTime::now_utc(),
1397            }))
1398            .await;
1399            let _ = approval.tx.send(true);
1400        }
1401    }
1402
1403    pub async fn record_pending_plan_exit(&self, pending: PendingPlanExit) {
1404        *self.pending_plan_exit.write().await = Some(pending.clone());
1405        self.emit(RoderEvent::PolicyExitPlanRequested(
1406            PolicyExitPlanRequested {
1407                thread_id: pending.thread_id,
1408                turn_id: pending.turn_id,
1409                request_id: pending.request_id,
1410                target_mode: pending.target_mode,
1411                plan_summary: pending.plan_summary,
1412                next_steps: pending.next_steps,
1413                timestamp: OffsetDateTime::now_utc(),
1414            },
1415        ))
1416        .await;
1417    }
1418
1419    pub async fn resolve_pending_plan_exit(
1420        &self,
1421        request_id: &str,
1422        approved: bool,
1423    ) -> anyhow::Result<Option<PendingPlanExit>> {
1424        let mut pending = self.pending_plan_exit.write().await;
1425        let Some(current) = pending.clone() else {
1426            return Ok(None);
1427        };
1428        if current.request_id != request_id {
1429            anyhow::bail!("pending plan exit request {request_id:?} was not found");
1430        }
1431        *pending = None;
1432        drop(pending);
1433
1434        let approved = approved && !current.is_expired(OffsetDateTime::now_utc());
1435        let resolved_mode = if approved {
1436            let mut cfg = self.config.write().await;
1437            let previous_mode = cfg.policy_mode;
1438            cfg.policy_mode = current.target_mode;
1439            drop(cfg);
1440            self.emit(RoderEvent::PolicyModeChanged(PolicyModeChanged {
1441                thread_id: current.thread_id.clone(),
1442                turn_id: Some(current.turn_id.clone()),
1443                previous_mode,
1444                new_mode: current.target_mode,
1445                reason: Some("approved plan exit".to_string()),
1446                timestamp: OffsetDateTime::now_utc(),
1447            }))
1448            .await;
1449            self.auto_resolve_pending_tool_approvals_for_mode(current.target_mode)
1450                .await;
1451            current.target_mode
1452        } else {
1453            self.status().await.policy_mode
1454        };
1455        self.emit_plan_exit_resolved(&current, approved, resolved_mode)
1456            .await;
1457        Ok(Some(current))
1458    }
1459
1460    pub async fn resolve_tool_approval(
1461        &self,
1462        approval_id: &str,
1463        approved: bool,
1464    ) -> anyhow::Result<bool> {
1465        let pending = self.pending_tool_approvals.lock().await.remove(approval_id);
1466        let Some(pending) = pending else {
1467            return Ok(false);
1468        };
1469        self.emit(RoderEvent::ApprovalResolved(ApprovalResolved {
1470            thread_id: pending.thread_id,
1471            turn_id: pending.turn_id,
1472            approval_id: approval_id.to_string(),
1473            tool_id: pending.tool_id,
1474            tool_name: pending.tool_name,
1475            approved,
1476            timestamp: OffsetDateTime::now_utc(),
1477        }))
1478        .await;
1479        let _ = pending.tx.send(approved);
1480        Ok(true)
1481    }
1482
1483    pub async fn request_app_server_tool_approval(
1484        &self,
1485        call: ToolCall,
1486        reason: Option<String>,
1487    ) -> anyhow::Result<bool> {
1488        let approval_id = call.id.clone();
1489        let (tx, rx) = oneshot::channel();
1490        self.pending_tool_approvals.lock().await.insert(
1491            approval_id.clone(),
1492            PendingToolApproval {
1493                thread_id: call.thread_id.clone(),
1494                turn_id: call.turn_id.clone(),
1495                tool_id: call.id.clone(),
1496                tool_name: call.name.clone(),
1497                call: call.clone(),
1498                tx,
1499            },
1500        );
1501        self.emit(RoderEvent::ApprovalRequested(ApprovalRequested {
1502            thread_id: call.thread_id.clone(),
1503            turn_id: call.turn_id.clone(),
1504            approval_id,
1505            tool_id: call.id.clone(),
1506            tool_name: call.name.clone(),
1507            reason,
1508            timestamp: OffsetDateTime::now_utc(),
1509        }))
1510        .await;
1511        Ok(rx.await.unwrap_or(false))
1512    }
1513
1514    /// Completes a pending host-executed tool call (`tools/resolve`). Returns false when the
1515    /// request id is unknown, already resolved, timed out, or cancelled by a turn interrupt.
1516    pub async fn resolve_external_tool_call(
1517        &self,
1518        request_id: &str,
1519        resolution: ExternalToolResolution,
1520    ) -> anyhow::Result<bool> {
1521        let mut calls = self.pending_external_tool_calls.lock().await;
1522        let Some(pending) = calls.get(request_id) else {
1523            return Ok(false);
1524        };
1525        self.emit_external_execution(RoderEvent::ExternalToolCallResolved(
1526            ExternalToolCallResolved {
1527                thread_id: pending.thread_id.clone(),
1528                turn_id: pending.turn_id.clone(),
1529                request_id: request_id.to_string(),
1530                tool_id: pending.tool_id.clone(),
1531                tool_name: pending.tool_name.clone(),
1532                outcome: ExternalToolCallOutcome::Resolved,
1533                is_error: resolution.is_error,
1534                timestamp: OffsetDateTime::now_utc(),
1535            },
1536        ))
1537        .await?;
1538        let Some(pending) = calls.remove(request_id) else {
1539            return Ok(false);
1540        };
1541        drop(calls);
1542        let _ = pending.tx.send(resolution);
1543        Ok(true)
1544    }
1545
1546    /// Drops every pending external tool call for the turn and reports them as cancelled so an
1547    /// interrupt does not leak requests waiting on `tools/resolve`.
1548    async fn cancel_pending_external_tool_calls_for_turn(&self, turn_id: &TurnId) {
1549        let cancelled = {
1550            let mut pending = self.pending_external_tool_calls.lock().await;
1551            let request_ids = pending
1552                .iter()
1553                .filter(|(_, call)| &call.turn_id == turn_id)
1554                .map(|(request_id, _)| request_id.clone())
1555                .collect::<Vec<_>>();
1556            request_ids
1557                .into_iter()
1558                .filter_map(|request_id| pending.remove(&request_id).map(|call| (request_id, call)))
1559                .collect::<Vec<_>>()
1560        };
1561        for (request_id, call) in cancelled {
1562            self.emit(RoderEvent::ExternalToolCallResolved(
1563                ExternalToolCallResolved {
1564                    thread_id: call.thread_id,
1565                    turn_id: call.turn_id,
1566                    request_id,
1567                    tool_id: call.tool_id,
1568                    tool_name: call.tool_name,
1569                    outcome: ExternalToolCallOutcome::Cancelled,
1570                    is_error: true,
1571                    timestamp: OffsetDateTime::now_utc(),
1572                },
1573            ))
1574            .await;
1575        }
1576    }
1577
1578    pub async fn resolve_user_input(
1579        &self,
1580        request_id: &str,
1581        answers: serde_json::Value,
1582    ) -> anyhow::Result<bool> {
1583        let pending = self.pending_user_inputs.lock().await.remove(request_id);
1584        let Some(pending) = pending else {
1585            return Ok(false);
1586        };
1587        self.emit(RoderEvent::UserInputResolved(UserInputResolved {
1588            thread_id: pending.thread_id,
1589            turn_id: pending.turn_id,
1590            request_id: request_id.to_string(),
1591            answers: answers.clone(),
1592            timestamp: OffsetDateTime::now_utc(),
1593        }))
1594        .await;
1595        let _ = pending.tx.send(answers);
1596        Ok(true)
1597    }
1598
1599    async fn emit_plan_exit_resolved(
1600        &self,
1601        current: &PendingPlanExit,
1602        approved: bool,
1603        resolved_mode: PolicyMode,
1604    ) {
1605        self.emit(RoderEvent::PolicyExitPlanResolved(PolicyExitPlanResolved {
1606            thread_id: current.thread_id.clone(),
1607            turn_id: current.turn_id.clone(),
1608            request_id: current.request_id.clone(),
1609            approved,
1610            target_mode: current.target_mode,
1611            resolved_mode,
1612            timestamp: OffsetDateTime::now_utc(),
1613        }))
1614        .await;
1615    }
1616
1617    pub async fn select_provider(
1618        &self,
1619        provider: String,
1620        model: Option<String>,
1621        reasoning: Option<String>,
1622    ) -> anyhow::Result<RuntimeConfig> {
1623        let next = self
1624            .preview_provider_selection(provider, model, reasoning)
1625            .await?;
1626        let mut cfg = self.config.write().await;
1627        *cfg = next;
1628        Ok(cfg.clone())
1629    }
1630
1631    pub async fn preview_provider_selection(
1632        &self,
1633        provider: String,
1634        model: Option<String>,
1635        reasoning: Option<String>,
1636    ) -> anyhow::Result<RuntimeConfig> {
1637        self.engine_for(&provider)?;
1638        let mut cfg = self.config.read().await.clone();
1639        cfg.default_provider = provider;
1640        if let Some(model) = model {
1641            cfg.default_model = model;
1642        }
1643        if let Some(reasoning) = reasoning {
1644            if reasoning == REASONING_NONE
1645                && !model_supports_reasoning(&cfg.default_model, &reasoning)
1646            {
1647                return Ok(cfg.clone());
1648            }
1649            validate_reasoning_effort(&cfg.default_model, &reasoning)?;
1650            cfg.reasoning = Some(reasoning);
1651        }
1652        Ok(cfg)
1653    }
1654
1655    pub async fn effective_reasoning(&self) -> String {
1656        let cfg = self.config.read().await;
1657        effective_reasoning_for_model(&cfg, &cfg.default_model)
1658    }
1659
1660    pub fn effective_reasoning_for_config(cfg: &RuntimeConfig) -> String {
1661        effective_reasoning_for_model(cfg, &cfg.default_model)
1662    }
1663
1664    pub async fn set_dynamic_workflow_effort(
1665        &self,
1666        effort_profile: DynamicWorkflowEffortProfile,
1667    ) -> RuntimeConfig {
1668        let mut cfg = self.config.write().await;
1669        cfg.dynamic_workflows.effort_profile = effort_profile;
1670        cfg.clone()
1671    }
1672
1673    pub async fn dynamic_workflow_trigger_decision(
1674        &self,
1675        message: &str,
1676    ) -> WorkflowTriggerDecision {
1677        let cfg = self.config.read().await;
1678        classify_workflow_trigger(message, &cfg.dynamic_workflows)
1679    }
1680
1681    pub async fn create_thread(&self, title: Option<String>) -> anyhow::Result<ThreadMetadata> {
1682        self.create_thread_with(CreateThreadRequest {
1683            title,
1684            workspace: self.workspace.display().to_string(),
1685            workspace_id: None,
1686            root_id: None,
1687            provider: None,
1688            model: None,
1689            selection_mode: None,
1690            tool_allowlist: Vec::new(),
1691            developer_instructions: None,
1692            external_tools: Vec::new(),
1693            runner: None,
1694        })
1695        .await
1696    }
1697
1698    /**
1699     * Resolves an explicit thread-runner selection into a persisted binding.
1700     * Fails fast at thread creation when the provider is missing or rejects
1701     * the destination, instead of surfacing the error on the first tool call.
1702     */
1703    async fn resolve_thread_runner_binding(
1704        &self,
1705        thread_id: &str,
1706        selection: ThreadRunnerSelection,
1707    ) -> anyhow::Result<ThreadRunnerBinding> {
1708        let provider = self
1709            .registry
1710            .remote_runner_providers
1711            .iter()
1712            .find(|provider| provider.id() == selection.provider_id)
1713            .cloned()
1714            .ok_or_else(|| {
1715                anyhow::anyhow!(
1716                    "remote runner provider {:?} is not installed",
1717                    selection.provider_id
1718                )
1719            })?;
1720        let workspace = selection.workspace.trim();
1721        anyhow::ensure!(
1722            std::path::Path::new(workspace).is_absolute(),
1723            "runner workspace must be an absolute path on the runner: {workspace:?}"
1724        );
1725        let mut read_roots = Vec::with_capacity(selection.read_roots.len());
1726        for read_root in &selection.read_roots {
1727            let trimmed = read_root.trim();
1728            anyhow::ensure!(
1729                std::path::Path::new(trimmed).is_absolute(),
1730                "runner read root must be an absolute path on the runner: {trimmed:?}"
1731            );
1732            read_roots.push(PathBuf::from(trimmed));
1733        }
1734        let destination = RunnerDestination {
1735            id: format!("thread-{thread_id}"),
1736            provider_id: selection.provider_id,
1737            config: selection.config,
1738            default_manifest: roder_api::remote_runner::RunnerManifest::default(),
1739        };
1740        provider.validate_destination(&destination).await?;
1741        Ok(ThreadRunnerBinding {
1742            destination,
1743            workspace: PathBuf::from(workspace),
1744            read_roots,
1745        })
1746    }
1747
1748    /**
1749     * Build a per-thread binding from a runtime-level destination (selected via
1750     * the TUI runner picker or config `default_destination`) when the
1751     * destination's provider advertises a default workspace. Returns `None` when
1752     * there is no destination or the provider opts out (no default workspace),
1753     * preserving the legacy behavior where such threads keep local tools.
1754     */
1755    async fn synthesize_runtime_runner_binding(
1756        &self,
1757        thread_id: &str,
1758        destination: Option<RunnerDestination>,
1759    ) -> anyhow::Result<Option<ThreadRunnerBinding>> {
1760        let Some(destination) = destination else {
1761            return Ok(None);
1762        };
1763        let Some(provider) = self
1764            .registry
1765            .remote_runner_providers
1766            .iter()
1767            .find(|provider| provider.id() == destination.provider_id)
1768        else {
1769            return Ok(None);
1770        };
1771        // An explicit workspace in the destination config wins over the
1772        // provider default; absence of both means the provider opts out.
1773        let workspace = destination
1774            .config
1775            .get("working_dir")
1776            .and_then(serde_json::Value::as_str)
1777            .map(str::to_string)
1778            .or_else(|| provider.default_workspace());
1779        let Some(workspace) = workspace else {
1780            return Ok(None);
1781        };
1782        let selection = ThreadRunnerSelection {
1783            provider_id: destination.provider_id.clone(),
1784            config: destination.config.clone(),
1785            workspace,
1786            read_roots: Vec::new(),
1787        };
1788        Ok(Some(
1789            self.resolve_thread_runner_binding(thread_id, selection)
1790                .await?,
1791        ))
1792    }
1793
1794    /// Validates a runner selection without creating a thread; the placeholder destination id only appears in error messages.
1795    pub async fn validate_thread_runner_selection(
1796        &self,
1797        selection: ThreadRunnerSelection,
1798    ) -> anyhow::Result<()> {
1799        self.resolve_thread_runner_binding("validate", selection)
1800            .await
1801            .map(|_| ())
1802    }
1803
1804    pub async fn create_thread_with(
1805        &self,
1806        req: CreateThreadRequest,
1807    ) -> anyhow::Result<ThreadMetadata> {
1808        tool_advertisement::validate_external_tool_names(&req.external_tools)?;
1809        let cfg = self.config.read().await.clone();
1810        let now = OffsetDateTime::now_utc();
1811        let workspace = validate_thread_workspace(&req.workspace)?;
1812        let provider = req.provider.unwrap_or(cfg.default_provider);
1813        let model = req.model.unwrap_or(cfg.default_model);
1814        let selection_mode = req
1815            .selection_mode
1816            .unwrap_or_else(|| ModelSelectionMode::manual(provider.clone(), model.clone(), None));
1817        let thread_id = uuid::Uuid::new_v4().to_string();
1818        let runner_binding = match req.runner {
1819            Some(selection) => Some(
1820                self.resolve_thread_runner_binding(&thread_id, selection)
1821                    .await?,
1822            ),
1823            // No explicit per-thread selection: if a runtime-level destination
1824            // is active and its provider advertises a default workspace, bind
1825            // the new thread so its coding tools route into the runner. This is
1826            // what makes a TUI/config-selected runner (e.g. Blaxel) actually
1827            // execute tools in the sandbox instead of locally. Providers that
1828            // return no default workspace stay unbound (legacy local-tools).
1829            None => {
1830                self.synthesize_runtime_runner_binding(
1831                    &thread_id,
1832                    cfg.remote_runner_destination.clone(),
1833                )
1834                .await?
1835            }
1836        };
1837        let runner_destination = runner_binding
1838            .as_ref()
1839            .map(|binding| binding.destination.clone())
1840            .or_else(|| cfg.remote_runner_destination.clone());
1841        let metadata = ThreadMetadata {
1842            thread_id,
1843            title: req.title,
1844            workspace,
1845            workspace_id: req.workspace_id,
1846            root_id: req.root_id,
1847            provider: Some(provider),
1848            model: Some(model),
1849            selection_mode: Some(selection_mode),
1850            tool_allowlist: req.tool_allowlist,
1851            developer_instructions: req.developer_instructions,
1852            external_tools: req.external_tools,
1853            runner_destination,
1854            runner_state: None,
1855            runner_binding,
1856            created_at: now,
1857            updated_at: now,
1858            message_count: 0,
1859            usage: None,
1860            parent_thread_id: None,
1861            forked_from_turn_id: None,
1862            workspace_fork: None,
1863        };
1864
1865        let metadata = if let Some(store) = &self.thread_store {
1866            store.create_thread(metadata).await?
1867        } else {
1868            metadata
1869        };
1870        self.emit(RoderEvent::ThreadCreated(ThreadCreated {
1871            thread_id: metadata.thread_id.clone(),
1872            timestamp: OffsetDateTime::now_utc(),
1873        }))
1874        .await;
1875        Ok(metadata)
1876    }
1877
1878    pub async fn list_threads(&self) -> anyhow::Result<Vec<ThreadMetadata>> {
1879        if let Some(store) = &self.thread_store {
1880            return store.list_threads().await;
1881        }
1882        Ok(Vec::new())
1883    }
1884
1885    pub async fn list_threads_page(
1886        &self,
1887        options: roder_api::thread::ThreadListOptions,
1888    ) -> anyhow::Result<roder_api::thread::ThreadListPage> {
1889        if let Some(store) = &self.thread_store {
1890            return store.list_threads_page(options).await;
1891        }
1892        Ok(roder_api::thread::ThreadListPage::default())
1893    }
1894
1895    pub async fn load_thread_metadata(
1896        &self,
1897        thread_id: &str,
1898    ) -> anyhow::Result<Option<roder_api::thread::ThreadMetadata>> {
1899        if let Some(store) = &self.thread_store {
1900            return store.load_thread_metadata(&thread_id.to_string()).await;
1901        }
1902        Ok(None)
1903    }
1904
1905    pub async fn archive_thread(&self, thread_id: &str) -> anyhow::Result<bool> {
1906        let archived = if let Some(store) = &self.thread_store {
1907            store.archive_thread(&thread_id.to_string()).await?
1908        } else {
1909            false
1910        };
1911        if archived {
1912            let workspace = self.config.read().await.workspace.clone();
1913            crate::hooks::run_lifecycle(
1914                self,
1915                &thread_id.to_string(),
1916                &SESSION_HOOK_TURN_ID.to_string(),
1917                workspace.as_deref(),
1918                "SessionEnd",
1919                Some("clear"),
1920                serde_json::json!({"reason": "archive"}),
1921            )
1922            .await;
1923            self.thread_item_cache
1924                .lock()
1925                .await
1926                .remove_thread(&thread_id.to_string());
1927            self.evict_runner_session(&thread_id.to_string()).await;
1928        }
1929        Ok(archived)
1930    }
1931
1932    pub async fn start_team(&self, req: TeamStartRequest) -> anyhow::Result<TeamState> {
1933        let cfg = self.config.read().await.clone();
1934        let workspace = self.workspace.display().to_string();
1935        let lead_thread_id = match req.lead_thread_id {
1936            Some(thread_id) => thread_id,
1937            None => {
1938                self.create_thread_with(CreateThreadRequest {
1939                    title: Some("Team lead".to_string()),
1940                    workspace: workspace.clone(),
1941                    workspace_id: None,
1942                    root_id: None,
1943                    provider: None,
1944                    model: None,
1945                    selection_mode: None,
1946                    tool_allowlist: Vec::new(),
1947                    developer_instructions: None,
1948                    external_tools: Vec::new(),
1949                    runner: None,
1950                })
1951                .await?
1952                .thread_id
1953            }
1954        };
1955        let team_id = uuid::Uuid::new_v4().to_string();
1956        let active_lead_turn_id = self.active_turn_for_thread(&lead_thread_id).await;
1957        let lead_selection = match active_lead_turn_id.as_ref() {
1958            Some(turn_id) => self
1959                .active_turn_selections
1960                .read()
1961                .await
1962                .get(turn_id)
1963                .cloned(),
1964            None => self.selection_mode_for_thread(&lead_thread_id).await?,
1965        };
1966        let lead_concrete_selection = lead_selection
1967            .as_ref()
1968            .map(ModelSelectionMode::concrete_selection);
1969        let mut lead = crate::teams::lead_member(
1970            lead_thread_id.clone(),
1971            lead_concrete_selection
1972                .as_ref()
1973                .map(|selection| selection.provider.clone())
1974                .or_else(|| Some(cfg.default_provider.clone())),
1975            lead_concrete_selection
1976                .as_ref()
1977                .map(|selection| selection.model.clone())
1978                .or_else(|| Some(cfg.default_model.clone())),
1979            cfg.policy_mode,
1980        );
1981        if let Some(turn_id) = active_lead_turn_id {
1982            lead.current_turn_id = Some(turn_id);
1983            lead.status = TeamMemberStatus::Running;
1984        }
1985        let mut members = vec![lead];
1986
1987        for (index, member) in req.members.into_iter().enumerate() {
1988            let thread = self
1989                .create_thread_with(CreateThreadRequest {
1990                    title: Some(member.name.clone()),
1991                    workspace: workspace.clone(),
1992                    workspace_id: None,
1993                    root_id: None,
1994                    provider: member.model_provider.clone(),
1995                    model: member.model.clone(),
1996                    selection_mode: None,
1997                    tool_allowlist: Vec::new(),
1998                    developer_instructions: None,
1999                    external_tools: Vec::new(),
2000                    runner: None,
2001                })
2002                .await?;
2003            let member_id = format!("member-{}", index + 1);
2004            let descriptor = crate::teams::teammate_member(
2005                member_id.clone(),
2006                member.name,
2007                thread.thread_id.clone(),
2008                member.model_provider.or(thread.provider),
2009                member.model.or(thread.model),
2010                cfg.policy_mode,
2011            );
2012            members.push(descriptor);
2013        }
2014
2015        let now = OffsetDateTime::now_utc();
2016        let team = self
2017            .teams
2018            .insert(TeamState {
2019                id: team_id.clone(),
2020                lead_thread_id: lead_thread_id.clone(),
2021                display_mode: req.display_mode,
2022                members,
2023                mailbox: Vec::new(),
2024                tasks: Vec::new(),
2025                created_at: now,
2026                updated_at: now,
2027            })
2028            .await?;
2029        self.emit(RoderEvent::TeamStarted(TeamStarted {
2030            team_id: team_id.clone(),
2031            lead_thread_id,
2032            display_mode: team.display_mode,
2033            members: team.members.clone(),
2034            tasks: team.tasks.clone(),
2035            timestamp: OffsetDateTime::now_utc(),
2036        }))
2037        .await;
2038        for member in team
2039            .members
2040            .iter()
2041            .filter(|member| member.role != roder_api::teams::TeamMemberRole::Lead)
2042        {
2043            self.emit(RoderEvent::TeamMemberStarted(TeamMemberStarted {
2044                team_id: team_id.clone(),
2045                member: member.clone(),
2046                timestamp: OffsetDateTime::now_utc(),
2047            }))
2048            .await;
2049        }
2050        Ok(team)
2051    }
2052
2053    pub async fn list_teams(&self) -> Vec<TeamState> {
2054        let active_thread_ids = self
2055            .active_turns
2056            .read()
2057            .await
2058            .values()
2059            .map(|handle| handle.thread_id.clone())
2060            .collect::<std::collections::HashSet<_>>();
2061        self.teams
2062            .list()
2063            .await
2064            .into_iter()
2065            .map(|team| runtime_local_team_view(team, &active_thread_ids))
2066            .collect()
2067    }
2068
2069    pub async fn read_team(&self, team_id: &str) -> Option<TeamState> {
2070        let active_thread_ids = self
2071            .active_turns
2072            .read()
2073            .await
2074            .values()
2075            .map(|handle| handle.thread_id.clone())
2076            .collect::<std::collections::HashSet<_>>();
2077        self.teams
2078            .get(team_id)
2079            .await
2080            .map(|team| runtime_local_team_view(team, &active_thread_ids))
2081    }
2082
2083    pub async fn start_team_member(
2084        &self,
2085        team_id: &str,
2086        req: TeamMemberStartRequest,
2087    ) -> anyhow::Result<TeamState> {
2088        self.start_team_member_with_selection(team_id, req, None)
2089            .await
2090    }
2091
2092    pub async fn message_team_member(
2093        self: &Arc<Self>,
2094        team_id: &str,
2095        member_id: &str,
2096        message: String,
2097    ) -> anyhow::Result<TurnId> {
2098        let team = self
2099            .read_team(team_id)
2100            .await
2101            .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2102        self.followup_team_member(&team.lead_thread_id, team_id, member_id, message)
2103            .await
2104    }
2105
2106    /// Queue a mailbox message without starting an idle agent. If the target is
2107    /// already running, the message is delivered at the next inference boundary.
2108    pub(crate) async fn queue_team_member_message(
2109        self: &Arc<Self>,
2110        caller_thread_id: &ThreadId,
2111        team_id: &str,
2112        member_id: &str,
2113        message: String,
2114    ) -> anyhow::Result<Option<TurnId>> {
2115        let _delivery_guard = self.agent_team_spawn_lock.lock().await;
2116        let team = self
2117            .read_team(team_id)
2118            .await
2119            .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2120        let member = team
2121            .members
2122            .iter()
2123            .find(|member| member.id == member_id)
2124            .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2125            .clone();
2126        if member.status == TeamMemberStatus::Closed {
2127            anyhow::bail!("subagent {} is closed", member.name);
2128        }
2129        let from_member_id = team
2130            .members
2131            .iter()
2132            .find(|candidate| candidate.thread_id == *caller_thread_id)
2133            .map(|candidate| candidate.id.clone());
2134        self.teams
2135            .append_mailbox_message(
2136                team_id,
2137                from_member_id,
2138                member_id.to_string(),
2139                TeamMailboxMessageKind::Message,
2140                message,
2141            )
2142            .await?;
2143        let Some(turn_id) = self.active_turn_for_thread(&member.thread_id).await else {
2144            return Ok(None);
2145        };
2146        self.deliver_pending_team_mailbox(team_id, member_id, member.thread_id, turn_id.clone())
2147            .await?;
2148        Ok(Some(turn_id))
2149    }
2150
2151    /// Deliver queued messages and start an idle agent, or steer its active turn.
2152    pub(crate) async fn followup_team_member(
2153        self: &Arc<Self>,
2154        caller_thread_id: &ThreadId,
2155        team_id: &str,
2156        member_id: &str,
2157        message: String,
2158    ) -> anyhow::Result<TurnId> {
2159        let _spawn_guard = self.agent_team_spawn_lock.lock().await;
2160        self.followup_team_member_locked(caller_thread_id, team_id, member_id, message)
2161            .await
2162    }
2163
2164    async fn followup_team_member_locked(
2165        self: &Arc<Self>,
2166        caller_thread_id: &ThreadId,
2167        team_id: &str,
2168        member_id: &str,
2169        message: String,
2170    ) -> anyhow::Result<TurnId> {
2171        let team = self
2172            .read_team(team_id)
2173            .await
2174            .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2175        let member = team
2176            .members
2177            .iter()
2178            .find(|member| member.id == member_id)
2179            .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2180            .clone();
2181        if member.status == TeamMemberStatus::Closed {
2182            anyhow::bail!("subagent {} is closed", member.name);
2183        }
2184        let from_member_id = team
2185            .members
2186            .iter()
2187            .find(|candidate| candidate.thread_id == *caller_thread_id)
2188            .map(|candidate| candidate.id.clone());
2189        self.teams
2190            .append_mailbox_message(
2191                team_id,
2192                from_member_id,
2193                member_id.to_string(),
2194                TeamMailboxMessageKind::NewTask,
2195                message,
2196            )
2197            .await?;
2198        if let Some(turn_id) = self.active_turn_for_thread(&member.thread_id).await {
2199            self.deliver_pending_team_mailbox(
2200                team_id,
2201                member_id,
2202                member.thread_id,
2203                turn_id.clone(),
2204            )
2205            .await?;
2206            return Ok(turn_id);
2207        }
2208
2209        self.ensure_codex_v2_team_capacity(&team, &member.id, None)
2210            .await?;
2211        let metadata = self.load_thread_metadata(&member.thread_id).await?;
2212        let inherited_context = self
2213            .team_member_turn_contexts
2214            .lock()
2215            .await
2216            .get(&member.thread_id)
2217            .cloned();
2218        let workspace = inherited_context
2219            .as_ref()
2220            .map(|context| context.workspace.clone())
2221            .or_else(|| metadata.as_ref().map(|metadata| metadata.workspace.clone()))
2222            .unwrap_or_else(|| self.workspace.display().to_string());
2223        let member_thread_id = member.thread_id;
2224        let turn_id = self
2225            .start_turn(StartTurnRequest {
2226                thread_id: member_thread_id.clone(),
2227                message: String::new(),
2228                images: Vec::new(),
2229                provider_override: member.model_provider,
2230                model_override: member.model,
2231                reasoning_override: metadata
2232                    .as_ref()
2233                    .and_then(|metadata| metadata.selection_mode.as_ref())
2234                    .and_then(ModelSelectionMode::reasoning)
2235                    .map(str::to_string),
2236                workspace,
2237                instructions: inherited_context
2238                    .as_ref()
2239                    .map(|context| context.instructions.clone())
2240                    .unwrap_or_else(crate::default_instructions),
2241                developer_context: inherited_context
2242                    .as_ref()
2243                    .and_then(|context| context.developer_context.clone()),
2244                task_ledger_required: false,
2245                service_tier_override: None,
2246            })
2247            .await?;
2248        Ok(turn_id)
2249    }
2250
2251    pub async fn set_team_member_policy_mode(
2252        &self,
2253        team_id: &str,
2254        member_id: &str,
2255        policy_mode: PolicyMode,
2256    ) -> anyhow::Result<TeamState> {
2257        self.teams
2258            .set_member_policy_mode(team_id, member_id, policy_mode)
2259            .await
2260    }
2261
2262    pub async fn interrupt_team_member(
2263        &self,
2264        team_id: &str,
2265        member_id: &str,
2266    ) -> anyhow::Result<Option<TurnId>> {
2267        let _interrupt_guard = self.agent_team_spawn_lock.lock().await;
2268        let team = self
2269            .read_team(team_id)
2270            .await
2271            .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2272        let member = team
2273            .members
2274            .iter()
2275            .find(|member| member.id == member_id)
2276            .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2277            .clone();
2278        if member.status != TeamMemberStatus::Running {
2279            return Ok(None);
2280        }
2281        let Some(turn_id) = member.current_turn_id.clone() else {
2282            return Ok(None);
2283        };
2284        if self
2285            .active_turn_for_thread(&member.thread_id)
2286            .await
2287            .as_ref()
2288            != Some(&turn_id)
2289        {
2290            return Ok(None);
2291        }
2292        self.interrupt_turn(member.thread_id.clone(), turn_id.clone())
2293            .await?;
2294        // Make queued mailbox work immediately eligible for the replacement turn while the
2295        // lifecycle lock still prevents a follow-up from starting. Delivery acknowledgements
2296        // are turn-owned, so a late acknowledgement from the aborted turn cannot steal a
2297        // reservation acquired by its replacement.
2298        self.teams
2299            .release_mailbox_reservations_for_turn(&turn_id)
2300            .await;
2301        self.teams
2302            .update_member(team_id, member_id, |member| {
2303                member.status = TeamMemberStatus::Interrupted;
2304                member.current_turn_id = None;
2305                member.terminal_error = None;
2306            })
2307            .await?;
2308        self.emit(RoderEvent::TeamMemberCompleted(TeamMemberCompleted {
2309            team_id: team_id.to_string(),
2310            member_id: member_id.to_string(),
2311            member_thread_id: member.thread_id,
2312            turn_id: Some(turn_id.clone()),
2313            status: TeamMemberStatus::Interrupted,
2314            final_message: member.final_message,
2315            error: None,
2316            timestamp: OffsetDateTime::now_utc(),
2317        }))
2318        .await;
2319        Ok(Some(turn_id))
2320    }
2321
2322    pub async fn close_team_member(
2323        &self,
2324        team_id: &str,
2325        member_id: &str,
2326    ) -> anyhow::Result<roder_api::teams::TeamMemberDescriptor> {
2327        let team = self
2328            .read_team(team_id)
2329            .await
2330            .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2331        let member = team
2332            .members
2333            .iter()
2334            .find(|member| member.id == member_id)
2335            .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2336            .clone();
2337        if member.role == roder_api::teams::TeamMemberRole::Lead {
2338            anyhow::bail!("team lead cannot be closed as a subagent");
2339        }
2340        let interrupted_turn_id = if member.status == TeamMemberStatus::Running {
2341            if let Some(turn_id) = member.current_turn_id.clone() {
2342                self.interrupt_turn(member.thread_id.clone(), turn_id.clone())
2343                    .await?;
2344                Some(turn_id)
2345            } else {
2346                None
2347            }
2348        } else {
2349            member.current_turn_id.clone()
2350        };
2351        let updated = self
2352            .teams
2353            .update_member(team_id, member_id, |member| {
2354                member.status = TeamMemberStatus::Closed;
2355                member.current_turn_id = None;
2356            })
2357            .await?;
2358        let closed = updated
2359            .members
2360            .iter()
2361            .find(|member| member.id == member_id)
2362            .cloned()
2363            .ok_or_else(|| anyhow::anyhow!("closed team member disappeared"))?;
2364        self.team_member_turn_contexts
2365            .lock()
2366            .await
2367            .remove(&closed.thread_id);
2368        self.emit(RoderEvent::TeamMemberCompleted(TeamMemberCompleted {
2369            team_id: team_id.to_string(),
2370            member_id: closed.id.clone(),
2371            member_thread_id: closed.thread_id.clone(),
2372            turn_id: interrupted_turn_id,
2373            status: TeamMemberStatus::Closed,
2374            final_message: closed.final_message.clone(),
2375            error: closed.terminal_error.clone(),
2376            timestamp: OffsetDateTime::now_utc(),
2377        }))
2378        .await;
2379        Ok(closed)
2380    }
2381
2382    pub async fn cleanup_team(&self, team_id: &str, force: bool) -> anyhow::Result<bool> {
2383        let Some(team) = self.read_team(team_id).await else {
2384            return Ok(false);
2385        };
2386        if !force
2387            && team
2388                .members
2389                .iter()
2390                .any(|member| member.status == TeamMemberStatus::Running)
2391        {
2392            anyhow::bail!("team {team_id:?} has active teammates; use forced cleanup");
2393        }
2394        if force {
2395            for member in team.members.iter().filter(|member| {
2396                member.role != roder_api::teams::TeamMemberRole::Lead
2397                    && member.status == TeamMemberStatus::Running
2398            }) {
2399                if let Some(turn_id) = member
2400                    .current_turn_id
2401                    .clone()
2402                    .or(self.active_turn_for_thread(&member.thread_id).await)
2403                {
2404                    let _ = self.interrupt_turn(member.thread_id.clone(), turn_id).await;
2405                }
2406            }
2407        }
2408        let removed = self.teams.remove(team_id).await?.is_some();
2409        if removed {
2410            let member_thread_ids = team
2411                .members
2412                .iter()
2413                .map(|member| member.thread_id.clone())
2414                .collect::<std::collections::HashSet<_>>();
2415            self.team_member_turn_contexts
2416                .lock()
2417                .await
2418                .retain(|thread_id, _| !member_thread_ids.contains(thread_id));
2419            self.emit(RoderEvent::TeamCleanupCompleted(TeamCleanupCompleted {
2420                team_id: team_id.to_string(),
2421                forced: force,
2422                timestamp: OffsetDateTime::now_utc(),
2423            }))
2424            .await;
2425        }
2426        Ok(removed)
2427    }
2428
2429    pub async fn effective_policy_mode_for_thread(&self, thread_id: &str) -> PolicyMode {
2430        if let Some(mode) = self.teams.policy_mode_for_thread(thread_id).await {
2431            return mode;
2432        }
2433        self.status().await.policy_mode
2434    }
2435
2436    async fn complete_team_member_turn_with_result(
2437        &self,
2438        thread_id: &ThreadId,
2439        turn_id: &TurnId,
2440        status: TeamMemberStatus,
2441        final_message: Option<String>,
2442        terminal_error: Option<String>,
2443    ) -> anyhow::Result<()> {
2444        let _delivery_guard = self.agent_team_spawn_lock.lock().await;
2445        let Some((team_id, member)) = self
2446            .teams
2447            .complete_member_turn(
2448                thread_id,
2449                turn_id,
2450                status,
2451                final_message.clone(),
2452                terminal_error.clone(),
2453            )
2454            .await?
2455        else {
2456            return Ok(());
2457        };
2458        if let Some(parent_thread_id) = member.parent_thread_id.as_ref()
2459            && let Some(team) = self.read_team(&team_id).await
2460            && let Some(parent) = team
2461                .members
2462                .iter()
2463                .find(|candidate| candidate.thread_id == *parent_thread_id)
2464        {
2465            let identity = member
2466                .agent_path
2467                .as_deref()
2468                .or(member.task_name.as_deref())
2469                .unwrap_or(&member.name);
2470            let mut report = format!("Agent {identity} finished with status {status:?}.");
2471            if let Some(message) = final_message.as_deref()
2472                && !message.trim().is_empty()
2473            {
2474                report.push_str("\n\nFinal result:\n");
2475                report.push_str(message);
2476            }
2477            if let Some(error) = terminal_error.as_deref()
2478                && !error.trim().is_empty()
2479            {
2480                report.push_str("\n\nTerminal error:\n");
2481                report.push_str(error);
2482            }
2483            let mailbox_appended = self
2484                .teams
2485                .append_mailbox_message(
2486                    &team_id,
2487                    Some(member.id.clone()),
2488                    parent.id.clone(),
2489                    TeamMailboxMessageKind::FinalAnswer,
2490                    report,
2491                )
2492                .await
2493                .is_ok();
2494            if mailbox_appended
2495                && let Some(parent_turn_id) = self.active_turn_for_thread(parent_thread_id).await
2496            {
2497                // The parent may finish between the active-turn lookup and enqueue. Its durable
2498                // mailbox entry remains pending for the next turn, while terminal observers must
2499                // still receive TeamMemberCompleted for this child.
2500                let _ = self
2501                    .deliver_pending_team_mailbox(
2502                        &team_id,
2503                        &parent.id,
2504                        parent_thread_id.clone(),
2505                        parent_turn_id,
2506                    )
2507                    .await;
2508            }
2509        }
2510        self.emit(RoderEvent::TeamMemberCompleted(TeamMemberCompleted {
2511            team_id,
2512            member_id: member.id,
2513            member_thread_id: member.thread_id,
2514            turn_id: Some(turn_id.clone()),
2515            status,
2516            final_message,
2517            error: terminal_error,
2518            timestamp: OffsetDateTime::now_utc(),
2519        }))
2520        .await;
2521        Ok(())
2522    }
2523
2524    /// Stops accepting new turns, requests interruption for every active turn,
2525    /// and waits for their runtime tasks to reach terminal cleanup up to
2526    /// `timeout`. Repeated calls are safe and continue draining the same set.
2527    pub async fn drain_active_turns(&self, timeout: std::time::Duration) -> RuntimeDrainOutcome {
2528        let started_at = tokio::time::Instant::now();
2529        let persistence_failures_before =
2530            self.lifecycle_persistence_failures.load(Ordering::Acquire);
2531        let turn_admission = self.turn_admission.lock().await;
2532        self.accepting_turns.store(false, Ordering::Release);
2533        let active = self
2534            .active_turns
2535            .read()
2536            .await
2537            .iter()
2538            .map(|(turn_id, handle)| (turn_id.clone(), handle.thread_id.clone()))
2539            .collect::<Vec<_>>();
2540        let draining = self
2541            .turn_drains
2542            .read()
2543            .await
2544            .iter()
2545            .map(|(turn_id, drain)| (turn_id.clone(), drain.thread_id.clone()))
2546            .collect::<Vec<_>>();
2547        drop(turn_admission);
2548
2549        let mut interrupted_turns = std::collections::BTreeMap::new();
2550        for (turn_id, thread_id) in active.iter().chain(draining.iter()) {
2551            interrupted_turns.insert(turn_id.clone(), thread_id.clone());
2552        }
2553        let interrupted_turn_ids = interrupted_turns.keys().cloned().collect::<Vec<_>>();
2554        for (turn_id, thread_id) in active {
2555            let _ = self
2556                .interrupt_turn_with_reason(thread_id, turn_id, TurnLifecycleReason::Shutdown)
2557                .await;
2558        }
2559
2560        let wait_for_empty = async {
2561            loop {
2562                let notified = self.active_turns_changed.notified();
2563                if self.active_turns.read().await.is_empty()
2564                    && self.turn_drains.read().await.is_empty()
2565                {
2566                    return;
2567                }
2568                notified.await;
2569            }
2570        };
2571        let outcome = if tokio::time::timeout(timeout, wait_for_empty).await.is_ok() {
2572            if self.lifecycle_persistence_failures.load(Ordering::Acquire)
2573                > persistence_failures_before
2574            {
2575                RuntimeDrainOutcome::PersistenceFailed {
2576                    interrupted_turn_ids,
2577                    remaining_turn_ids: Vec::new(),
2578                }
2579            } else {
2580                RuntimeDrainOutcome::Clean {
2581                    interrupted_turn_ids,
2582                }
2583            }
2584        } else {
2585            let active_remaining = self
2586                .active_turns
2587                .read()
2588                .await
2589                .iter()
2590                .map(|(turn_id, handle)| (turn_id.clone(), handle.drain.clone()))
2591                .collect::<Vec<_>>();
2592            let draining_remaining = self
2593                .turn_drains
2594                .read()
2595                .await
2596                .iter()
2597                .map(|(turn_id, drain)| (turn_id.clone(), drain.clone()))
2598                .collect::<Vec<_>>();
2599            let remaining = active_remaining
2600                .into_iter()
2601                .chain(draining_remaining)
2602                .collect::<std::collections::BTreeMap<_, _>>();
2603            let remaining_turn_ids = remaining.keys().cloned().collect::<Vec<_>>();
2604            for turn_id in &remaining_turn_ids {
2605                let handle = remaining
2606                    .get(turn_id)
2607                    .expect("remaining turn ID must have a drain handle");
2608                self.record_turn_lifecycle(
2609                    handle.thread_id.clone(),
2610                    turn_id.clone(),
2611                    TurnLifecycleState::InterruptRequested,
2612                    TurnCleanupState::TimedOut,
2613                    Some(TurnLifecycleReason::Shutdown),
2614                )
2615                .await;
2616            }
2617            if self.lifecycle_persistence_failures.load(Ordering::Acquire)
2618                > persistence_failures_before
2619            {
2620                RuntimeDrainOutcome::PersistenceFailed {
2621                    interrupted_turn_ids,
2622                    remaining_turn_ids,
2623                }
2624            } else {
2625                RuntimeDrainOutcome::DeadlineExceeded {
2626                    interrupted_turn_ids,
2627                    remaining_turn_ids,
2628                }
2629            }
2630        };
2631        self.record_lifecycle_drain_outcome(started_at, &outcome);
2632        outcome
2633    }
2634
2635    /// Enables turn submission after a prior drain attempt. This is intended
2636    /// for embedders that keep a runtime alive after a bounded drain timeout.
2637    pub fn resume_accepting_turns(&self) {
2638        self.accepting_turns.store(true, Ordering::Release);
2639    }
2640
2641    pub async fn turn_lifecycle_snapshot(
2642        &self,
2643        thread_id: &ThreadId,
2644    ) -> anyhow::Result<TurnLifecycleSnapshot> {
2645        let Some(store) = &self.thread_store else {
2646            return Ok(TurnLifecycleSnapshot::default());
2647        };
2648        let extension_states = store.load_extension_states(thread_id).await?;
2649        Ok(turn_lifecycle_snapshot(&extension_states))
2650    }
2651
2652    pub async fn load_thread(
2653        &self,
2654        thread_id: &ThreadId,
2655    ) -> anyhow::Result<Option<ThreadSnapshot>> {
2656        let loaded = if let Some(store) = &self.thread_store {
2657            store.load_thread(thread_id).await?
2658        } else {
2659            None
2660        };
2661        if let Some(snapshot) = loaded.as_ref() {
2662            let live_turn_ids = self
2663                .active_turns
2664                .read()
2665                .await
2666                .keys()
2667                .cloned()
2668                .chain(self.turn_drains.read().await.keys().cloned())
2669                .collect::<std::collections::HashSet<_>>();
2670            let lifecycle = turn_lifecycle_snapshot(&snapshot.extension_states);
2671            for record in lifecycle.records.into_iter().filter(|record| {
2672                record.state.requires_recovery() && !live_turn_ids.contains(&record.turn_id)
2673            }) {
2674                self.lifecycle_restart_reconciliations
2675                    .fetch_add(1, Ordering::AcqRel);
2676                self.record_turn_lifecycle(
2677                    record.thread_id,
2678                    record.turn_id,
2679                    TurnLifecycleState::RecoveryNeeded,
2680                    TurnCleanupState::Unknown,
2681                    Some(TurnLifecycleReason::RuntimeRestart),
2682                )
2683                .await;
2684            }
2685            self.emit(RoderEvent::ThreadLoaded(ThreadLoaded {
2686                thread_id: thread_id.clone(),
2687                timestamp: OffsetDateTime::now_utc(),
2688            }))
2689            .await;
2690        }
2691        Ok(loaded)
2692    }
2693
2694    pub async fn workspace_for_thread(&self, thread_id: &ThreadId) -> anyhow::Result<String> {
2695        if let Some(store) = &self.thread_store {
2696            let snapshot = store
2697                .load_thread(thread_id)
2698                .await?
2699                .ok_or_else(|| anyhow::anyhow!("thread not found: {thread_id}"));
2700            match snapshot {
2701                Ok(snapshot) => {
2702                    if let Some(metadata) = snapshot.metadata {
2703                        // Fork-backed threads fail closed before any write
2704                        // when their workspace was removed out-of-band.
2705                        if let Some(fork) = &metadata.workspace_fork
2706                            && fork.status == roder_api::forks::ForkStatus::Active
2707                            && !std::path::Path::new(&metadata.workspace).is_dir()
2708                        {
2709                            anyhow::bail!(
2710                                "workspace fork {} is missing its workspace at {}; restore it or \
2711                                 remove the fork before running turns in this thread",
2712                                fork.id,
2713                                metadata.workspace
2714                            );
2715                        }
2716                        return Ok(metadata.workspace);
2717                    }
2718                    eprintln!(
2719                        "thread metadata missing while resolving workspace for {thread_id}; falling back to runtime workspace"
2720                    );
2721                }
2722                Err(err) => {
2723                    eprintln!(
2724                        "thread missing while resolving workspace for {thread_id}: {err}; falling back to runtime workspace"
2725                    );
2726                }
2727            }
2728        }
2729        Ok(self.workspace.display().to_string())
2730    }
2731
2732    pub(crate) async fn selection_mode_for_thread(
2733        &self,
2734        thread_id: &ThreadId,
2735    ) -> anyhow::Result<Option<ModelSelectionMode>> {
2736        let Some(store) = &self.thread_store else {
2737            return Ok(None);
2738        };
2739        Ok(store
2740            .load_thread(thread_id)
2741            .await?
2742            .and_then(|snapshot| snapshot.metadata)
2743            .and_then(|metadata| {
2744                metadata
2745                    .selection_mode
2746                    .or_else(|| match (metadata.provider, metadata.model) {
2747                        (Some(provider), Some(model)) => {
2748                            Some(ModelSelectionMode::manual(provider, model, None))
2749                        }
2750                        _ => None,
2751                    })
2752            }))
2753    }
2754
2755    /// Concrete provider/model for configured-role subagents (`task`,
2756    /// `agent_swarm`). Prefers the live parent turn selection so children stay
2757    /// on SuperGrok/Grok (or whatever the lead is using) instead of process
2758    /// startup defaults such as openai/gpt-5.5.
2759    pub(crate) async fn parent_model_selection_for_subagents(
2760        &self,
2761        thread_id: &ThreadId,
2762        turn_id: &TurnId,
2763    ) -> Option<roder_api::inference::ModelSelection> {
2764        if let Some(selection) = self
2765            .active_turn_selections
2766            .read()
2767            .await
2768            .get(turn_id)
2769            .cloned()
2770        {
2771            return Some(selection.concrete_selection());
2772        }
2773        if let Ok(Some(selection)) = self.selection_mode_for_thread(thread_id).await {
2774            return Some(selection.concrete_selection());
2775        }
2776        let cfg = self.status().await;
2777        if cfg.default_provider.trim().is_empty() || cfg.default_model.trim().is_empty() {
2778            return None;
2779        }
2780        Some(roder_api::inference::ModelSelection {
2781            provider: cfg.default_provider,
2782            model: cfg.default_model,
2783        })
2784    }
2785
2786    /// Per-thread tool allowlist, developer instructions, and external tools persisted at thread creation.
2787    pub(crate) async fn thread_turn_overrides(
2788        &self,
2789        thread_id: &ThreadId,
2790    ) -> anyhow::Result<ThreadTurnOverrides> {
2791        let Some(store) = &self.thread_store else {
2792            return Ok(ThreadTurnOverrides::default());
2793        };
2794        Ok(store
2795            .load_thread_metadata(thread_id)
2796            .await?
2797            .map(|metadata| ThreadTurnOverrides {
2798                tool_allowlist: metadata.tool_allowlist,
2799                developer_instructions: metadata.developer_instructions,
2800                external_tools: metadata.external_tools,
2801            })
2802            .unwrap_or_default())
2803    }
2804
2805    pub async fn set_thread_selection_mode(
2806        &self,
2807        thread_id: &ThreadId,
2808        selection_mode: ModelSelectionMode,
2809    ) -> anyhow::Result<()> {
2810        let Some(store) = &self.thread_store else {
2811            return Ok(());
2812        };
2813        let Some(snapshot) = store.load_thread(thread_id).await? else {
2814            anyhow::bail!("thread not found: {thread_id}");
2815        };
2816        let Some(mut metadata) = snapshot.metadata else {
2817            return Ok(());
2818        };
2819        let concrete = selection_mode.concrete_selection();
2820        metadata.provider = Some(concrete.provider);
2821        metadata.model = Some(concrete.model);
2822        metadata.selection_mode = Some(selection_mode);
2823        metadata.updated_at = OffsetDateTime::now_utc();
2824        store.update_thread_metadata(metadata).await?;
2825        Ok(())
2826    }
2827
2828    async fn runner_session_for_thread(
2829        &self,
2830        thread_id: &ThreadId,
2831    ) -> anyhow::Result<Option<(RunnerDestination, Arc<dyn RemoteRunnerSession>)>> {
2832        let metadata = if let Some(store) = &self.thread_store {
2833            store.load_thread_metadata(thread_id).await?
2834        } else {
2835            None
2836        };
2837        // An explicit per-thread binding wins over the runtime-level destination.
2838        let destination = metadata
2839            .as_ref()
2840            .and_then(|metadata| metadata.runner_binding.as_ref())
2841            .map(|binding| binding.destination.clone())
2842            .or(self.config.read().await.remote_runner_destination.clone());
2843        let Some(destination) = destination else {
2844            self.evict_runner_session(thread_id).await;
2845            return Ok(None);
2846        };
2847        let provider = self
2848            .registry
2849            .remote_runner_providers
2850            .iter()
2851            .find(|provider| provider.id() == destination.provider_id)
2852            .cloned()
2853            .ok_or_else(|| {
2854                anyhow::anyhow!(
2855                    "remote runner provider {:?} is not installed",
2856                    destination.provider_id
2857                )
2858            })?;
2859        let persisted_state = metadata.and_then(|metadata| metadata.runner_state);
2860        let slot = {
2861            let mut sessions = self.runner_sessions.lock().await;
2862            match sessions.get(thread_id) {
2863                Some(slot) if slot.matches(&destination) => slot.clone(),
2864                _ => {
2865                    let slot = Arc::new(RunnerSessionSlot::empty(&destination));
2866                    sessions.insert(thread_id.clone(), slot.clone());
2867                    slot
2868                }
2869            }
2870        };
2871
2872        loop {
2873            let initialized = slot.initialized.notified();
2874            let mut state = slot.state.lock().await;
2875            match &*state {
2876                RunnerSessionSlotState::Ready(cached) => {
2877                    return Ok(Some((cached.destination.clone(), cached.session.clone())));
2878                }
2879                RunnerSessionSlotState::Failed(error) => {
2880                    return Err(anyhow::anyhow!(error.to_string()));
2881                }
2882                RunnerSessionSlotState::Initializing => {
2883                    drop(state);
2884                    initialized.await;
2885                }
2886                RunnerSessionSlotState::Empty => {
2887                    *state = RunnerSessionSlotState::Initializing;
2888                    drop(state);
2889                    self.spawn_runner_session_initialization(
2890                        thread_id.clone(),
2891                        destination.clone(),
2892                        provider.clone(),
2893                        persisted_state.clone(),
2894                        slot.clone(),
2895                    );
2896                }
2897            }
2898        }
2899    }
2900
2901    fn spawn_runner_session_initialization(
2902        &self,
2903        thread_id: ThreadId,
2904        destination: RunnerDestination,
2905        provider: Arc<dyn RemoteRunnerProvider>,
2906        persisted_state: Option<RunnerSessionState>,
2907        slot: Arc<RunnerSessionSlot>,
2908    ) {
2909        let thread_store = self.thread_store.clone();
2910        let runner_sessions = self.runner_sessions.clone();
2911        // The task intentionally outlives the turn that first requested the
2912        // workspace. Interrupting that turn must not strand the slot in
2913        // `Initializing` and deadlock every later tool or lifecycle call.
2914        drop(tokio::spawn(async move {
2915            let initialized = async {
2916                let session = if let Some(state) = persisted_state
2917                    && state.provider_id == destination.provider_id
2918                    && state.destination_id == destination.id
2919                {
2920                    match provider.resume_session(state).await {
2921                        Ok(session) => session,
2922                        Err(_) => provider.create_session(destination.clone()).await?,
2923                    }
2924                } else {
2925                    provider.create_session(destination.clone()).await?
2926                };
2927                let resolved = (destination.clone(), session.clone());
2928                // Persist before releasing the singleflight or dispatching
2929                // the first tool. A later process can rejoin even if the turn
2930                // that requested initialization has already been interrupted.
2931                Self::persist_runner_state_in_store(
2932                    thread_store.as_ref(),
2933                    &thread_id,
2934                    Some(&resolved),
2935                )
2936                .await?;
2937                Ok::<_, anyhow::Error>(CachedRunnerSession {
2938                    destination,
2939                    session,
2940                })
2941            }
2942            .await;
2943
2944            match initialized {
2945                Ok(cached) => {
2946                    *slot.state.lock().await = RunnerSessionSlotState::Ready(cached);
2947                    slot.initialized.notify_waiters();
2948                }
2949                Err(error) => {
2950                    let message: Arc<str> = Arc::from(format!("{error:#}"));
2951                    *slot.state.lock().await = RunnerSessionSlotState::Failed(message);
2952                    slot.initialized.notify_waiters();
2953                    let mut sessions = runner_sessions.lock().await;
2954                    let is_current = sessions
2955                        .get(&thread_id)
2956                        .is_some_and(|current| Arc::ptr_eq(current, &slot));
2957                    if is_current {
2958                        sessions.remove(&thread_id);
2959                    }
2960                }
2961            }
2962        }));
2963    }
2964
2965    async fn evict_runner_session(&self, thread_id: &ThreadId) {
2966        self.runner_sessions.lock().await.remove(thread_id);
2967    }
2968
2969    async fn cache_runner_session(
2970        &self,
2971        thread_id: &ThreadId,
2972        destination: RunnerDestination,
2973        session: Arc<dyn RemoteRunnerSession>,
2974    ) {
2975        let cached = CachedRunnerSession {
2976            destination,
2977            session,
2978        };
2979        self.runner_sessions.lock().await.insert(
2980            thread_id.clone(),
2981            Arc::new(RunnerSessionSlot::ready(cached)),
2982        );
2983    }
2984
2985    /// Read a thread's explicit runner binding without creating or resuming a
2986    /// session. Tool routing uses this before local previews so runner-backed
2987    /// and fail-closed hosted calls never inspect the host workspace.
2988    pub(crate) async fn runner_binding_for_thread(
2989        &self,
2990        thread_id: &ThreadId,
2991    ) -> anyhow::Result<Option<ThreadRunnerBinding>> {
2992        let Some(store) = &self.thread_store else {
2993            return Ok(None);
2994        };
2995        Ok(store
2996            .load_thread_metadata(thread_id)
2997            .await?
2998            .and_then(|metadata| metadata.runner_binding))
2999    }
3000
3001    /// Resolve a previously-read binding into a live remote workspace. The
3002    /// session remains lazy: callers invoke this only after policy approval.
3003    pub(crate) async fn remote_workspace_for_binding(
3004        &self,
3005        thread_id: &ThreadId,
3006        binding: ThreadRunnerBinding,
3007    ) -> anyhow::Result<Arc<RemoteWorkspace>> {
3008        let session = self
3009            .runner_session_for_thread(thread_id)
3010            .await?
3011            .map(|(_, session)| session)
3012            .ok_or_else(|| {
3013                anyhow::anyhow!("runner-bound thread {thread_id} has no runner session")
3014            })?;
3015        Ok(Arc::new(RemoteWorkspace {
3016            session,
3017            root: binding.workspace,
3018            read_roots: binding.read_roots,
3019        }))
3020    }
3021
3022    pub(crate) async fn runner_tool_execution_lock(
3023        &self,
3024        session: &dyn RemoteRunnerSession,
3025    ) -> Arc<RunnerToolExecutionMutex> {
3026        let state = session.state();
3027        let key = (state.provider_id, state.session_id);
3028        let mut locks = self.runner_tool_execution_locks.lock().await;
3029        if let Some(lock) = locks.get(&key).and_then(Weak::upgrade) {
3030            return lock;
3031        }
3032
3033        locks.retain(|_, lock| lock.strong_count() > 0);
3034        let lock = Arc::new(Mutex::new(()));
3035        locks.insert(key, Arc::downgrade(&lock));
3036        lock
3037    }
3038
3039    async fn persist_runner_state(
3040        &self,
3041        thread_id: &ThreadId,
3042        runner: Option<&(RunnerDestination, Arc<dyn RemoteRunnerSession>)>,
3043    ) -> anyhow::Result<()> {
3044        Self::persist_runner_state_in_store(self.thread_store.as_ref(), thread_id, runner).await
3045    }
3046
3047    async fn persist_runner_state_in_store(
3048        store: Option<&Arc<dyn ThreadStore>>,
3049        thread_id: &ThreadId,
3050        runner: Option<&(RunnerDestination, Arc<dyn RemoteRunnerSession>)>,
3051    ) -> anyhow::Result<()> {
3052        let Some((destination, session)) = runner else {
3053            return Ok(());
3054        };
3055        let Some(store) = store else {
3056            return Ok(());
3057        };
3058        let Some(snapshot) = store.load_thread(thread_id).await? else {
3059            return Ok(());
3060        };
3061        let Some(mut metadata) = snapshot.metadata else {
3062            return Ok(());
3063        };
3064        metadata.runner_destination = Some(destination.clone());
3065        metadata.runner_state = Some(session.state());
3066        metadata.updated_at = OffsetDateTime::now_utc();
3067        store.update_thread_metadata(metadata).await?;
3068        Ok(())
3069    }
3070
3071    fn remote_runner_provider_by_id(
3072        &self,
3073        provider_id: &str,
3074    ) -> Option<Arc<dyn RemoteRunnerProvider>> {
3075        self.registry
3076            .remote_runner_providers
3077            .iter()
3078            .find(|provider| provider.id() == provider_id)
3079            .cloned()
3080    }
3081
3082    /// Resolve the live runner session for a thread along with its provider,
3083    /// failing clearly when the thread is not runner-bound.
3084    async fn thread_runner_session(
3085        &self,
3086        thread_id: &ThreadId,
3087    ) -> anyhow::Result<(
3088        RunnerDestination,
3089        Arc<dyn RemoteRunnerProvider>,
3090        Arc<dyn RemoteRunnerSession>,
3091    )> {
3092        let Some((destination, session)) = self.runner_session_for_thread(thread_id).await? else {
3093            anyhow::bail!("thread {thread_id} is not bound to a remote runner");
3094        };
3095        let provider = self
3096            .remote_runner_provider_by_id(&destination.provider_id)
3097            .ok_or_else(|| {
3098                anyhow::anyhow!(
3099                    "remote runner provider {:?} is not installed",
3100                    destination.provider_id
3101                )
3102            })?;
3103        Ok((destination, provider, session))
3104    }
3105
3106    /// Pause a runner-bound thread's session toward standby and persist the
3107    /// post-pause state. Errors if the provider is not pausable.
3108    pub async fn pause_thread_runner(
3109        &self,
3110        thread_id: &ThreadId,
3111    ) -> anyhow::Result<RunnerSessionState> {
3112        let (destination, provider, session) = self.thread_runner_session(thread_id).await?;
3113        anyhow::ensure!(
3114            provider.capabilities().pausable,
3115            "remote runner provider {:?} does not support pausing",
3116            destination.provider_id
3117        );
3118        let state = session.pause().await?;
3119        self.persist_runner_state(thread_id, Some(&(destination, session)))
3120            .await?;
3121        Ok(state)
3122    }
3123
3124    /// Resume (wake) a runner-bound thread's paused session and persist state.
3125    pub async fn resume_thread_runner(
3126        &self,
3127        thread_id: &ThreadId,
3128    ) -> anyhow::Result<RunnerSessionState> {
3129        let (destination, _provider, session) = self.thread_runner_session(thread_id).await?;
3130        let state = session.resume().await?;
3131        self.persist_runner_state(thread_id, Some(&(destination, session)))
3132            .await?;
3133        Ok(state)
3134    }
3135
3136    /// Detach a runner-bound thread's session: persist the durable, rejoinable
3137    /// state and leave the remote sandbox alive. Errors if not detachable.
3138    pub async fn detach_thread_runner(
3139        &self,
3140        thread_id: &ThreadId,
3141    ) -> anyhow::Result<RunnerSessionState> {
3142        let (destination, provider, session) = self.thread_runner_session(thread_id).await?;
3143        anyhow::ensure!(
3144            provider.capabilities().detachable,
3145            "remote runner provider {:?} does not support detaching",
3146            destination.provider_id
3147        );
3148        let state = session.detach().await?;
3149        // Persist the detached state explicitly so a later turn or process
3150        // rejoins the same sandbox instead of provisioning a new one.
3151        if let Some(store) = &self.thread_store
3152            && let Some(snapshot) = store.load_thread(thread_id).await?
3153            && let Some(mut metadata) = snapshot.metadata
3154        {
3155            metadata.runner_destination = Some(destination);
3156            metadata.runner_state = Some(state.clone());
3157            metadata.updated_at = OffsetDateTime::now_utc();
3158            store.update_thread_metadata(metadata).await?;
3159        }
3160        self.evict_runner_session(thread_id).await;
3161        Ok(state)
3162    }
3163
3164    /// Rejoin a previously created sandbox from a thread's persisted runner
3165    /// state without provisioning a new one. An optional `sandbox` overrides the
3166    /// persisted sandbox name (recovery by name). Persists refreshed state.
3167    pub async fn rejoin_thread_runner(
3168        &self,
3169        thread_id: &ThreadId,
3170        sandbox: Option<String>,
3171    ) -> anyhow::Result<RunnerSessionState> {
3172        let store = self
3173            .thread_store
3174            .as_ref()
3175            .ok_or_else(|| anyhow::anyhow!("thread store is required to rejoin a runner"))?;
3176        let metadata = store
3177            .load_thread_metadata(thread_id)
3178            .await?
3179            .ok_or_else(|| anyhow::anyhow!("thread {thread_id} has no metadata"))?;
3180        let destination = metadata
3181            .runner_binding
3182            .as_ref()
3183            .map(|binding| binding.destination.clone())
3184            .or_else(|| metadata.runner_destination.clone())
3185            .ok_or_else(|| anyhow::anyhow!("thread {thread_id} is not bound to a remote runner"))?;
3186        let mut state = metadata
3187            .runner_state
3188            .clone()
3189            .ok_or_else(|| anyhow::anyhow!("thread {thread_id} has no persisted runner state"))?;
3190        if let Some(sandbox) = sandbox
3191            && let Some(object) = state.metadata.as_object_mut()
3192        {
3193            object.insert("sandbox_name".to_string(), serde_json::Value::from(sandbox));
3194        }
3195        let provider = self
3196            .remote_runner_provider_by_id(&destination.provider_id)
3197            .ok_or_else(|| {
3198                anyhow::anyhow!(
3199                    "remote runner provider {:?} is not installed",
3200                    destination.provider_id
3201                )
3202            })?;
3203        let session = provider.rejoin_session(state).await?;
3204        self.persist_runner_state(thread_id, Some(&(destination.clone(), session.clone())))
3205            .await?;
3206        self.cache_runner_session(thread_id, destination, session.clone())
3207            .await;
3208        Ok(session.state())
3209    }
3210
3211    pub(crate) async fn record_thread_usage_metadata(
3212        &self,
3213        thread_id: &ThreadId,
3214        usage: &TokenUsage,
3215    ) -> anyhow::Result<()> {
3216        if usage.is_empty() {
3217            return Ok(());
3218        }
3219        let Some(store) = &self.thread_store else {
3220            return Ok(());
3221        };
3222        let Some(snapshot) = store.load_thread(thread_id).await? else {
3223            return Ok(());
3224        };
3225        let Some(mut metadata) = snapshot.metadata else {
3226            return Ok(());
3227        };
3228        metadata
3229            .usage
3230            .get_or_insert_with(ThreadUsageMetadata::default)
3231            .add_token_usage(usage);
3232        metadata.updated_at = OffsetDateTime::now_utc();
3233        store.update_thread_metadata(metadata).await?;
3234        Ok(())
3235    }
3236
3237    pub fn start_turn(
3238        self: &Arc<Self>,
3239        mut req: StartTurnRequest,
3240    ) -> BoxFuture<'_, anyhow::Result<TurnId>> {
3241        Box::pin(async move {
3242            let _thread_admission = self.thread_admission(&req.thread_id).await;
3243            let turn_admission = self.turn_admission.lock().await;
3244            self.ensure_execution_authority()?;
3245            anyhow::ensure!(
3246                self.accepting_turns.load(Ordering::Acquire),
3247                "runtime is quiescing and cannot accept new turns"
3248            );
3249            req.workspace = validate_thread_workspace(&req.workspace)?;
3250            let team_member = self.teams.member_for_thread(&req.thread_id).await;
3251            let cfg = self.config.read().await.clone();
3252            let provider = req
3253                .provider_override
3254                .clone()
3255                .unwrap_or_else(|| cfg.default_provider.clone());
3256            self.engine_for(&provider)?;
3257            let turn_id = uuid::Uuid::new_v4().to_string();
3258            let mut initial_mailbox_ack = None;
3259            if let Some((team_id, member)) = &team_member {
3260                let pending = self
3261                    .teams
3262                    .reserve_pending_mailbox_messages(team_id, &member.id, &turn_id)
3263                    .await?;
3264                if !pending.is_empty()
3265                    && let Some(team) = self.read_team(team_id).await
3266                {
3267                    let mailbox = format_mailbox_messages(&team, &pending);
3268                    req.message = if req.message.trim().is_empty() {
3269                        mailbox
3270                    } else {
3271                        format!("{mailbox}\n\n[Direct task input]\n{}", req.message)
3272                    };
3273                    initial_mailbox_ack = Some(MailboxDeliveryAck {
3274                        team_id: team_id.clone(),
3275                        message_ids: pending.iter().map(|message| message.id.clone()).collect(),
3276                    });
3277                }
3278            }
3279            let (abort_handle, abort_registration) = AbortHandle::new_pair();
3280            let drain = Arc::new(TurnDrainHandle {
3281                thread_id: req.thread_id.clone(),
3282                interrupt_requested: AtomicBool::new(false),
3283                interrupt_reason: Mutex::new(None),
3284                completed: AtomicBool::new(false),
3285                completed_notify: Notify::new(),
3286            });
3287            let (steer_changed, steering) = tokio::sync::watch::channel(0);
3288            let active = ActiveTurnHandle {
3289                thread_id: req.thread_id.clone(),
3290                abort: abort_handle,
3291                steers: Arc::new(Mutex::new(Vec::new())),
3292                steer_changed,
3293                drain,
3294            };
3295            self.active_turns
3296                .write()
3297                .await
3298                .insert(turn_id.clone(), active);
3299            self.record_turn_lifecycle(
3300                req.thread_id.clone(),
3301                turn_id.clone(),
3302                TurnLifecycleState::Running,
3303                TurnCleanupState::NotRequested,
3304                None,
3305            )
3306            .await;
3307            self.active_turn_contexts.write().await.insert(
3308                turn_id.clone(),
3309                InheritedTurnContext {
3310                    workspace: req.workspace.clone(),
3311                    instructions: req.instructions.clone(),
3312                    developer_context: req.developer_context.clone(),
3313                },
3314            );
3315            if let Some((team_id, member)) = team_member {
3316                let updated = match self
3317                    .teams
3318                    .update_member(&team_id, &member.id, |member| {
3319                        member.current_turn_id = Some(turn_id.clone());
3320                        member.status = TeamMemberStatus::Running;
3321                        member.final_message = None;
3322                        member.terminal_error = None;
3323                    })
3324                    .await
3325                {
3326                    Ok(updated) => updated,
3327                    Err(error) => {
3328                        self.active_turns.write().await.remove(&turn_id);
3329                        self.active_turn_contexts.write().await.remove(&turn_id);
3330                        self.teams
3331                            .release_mailbox_reservations_for_turn(&turn_id)
3332                            .await;
3333                        return Err(error);
3334                    }
3335                };
3336                if let Some(member) = updated
3337                    .members
3338                    .into_iter()
3339                    .find(|candidate| candidate.id == member.id)
3340                {
3341                    self.emit(RoderEvent::TeamMemberStatusChanged(
3342                        TeamMemberStatusChanged {
3343                            team_id,
3344                            member_id: member.id,
3345                            member_thread_id: member.thread_id,
3346                            status: TeamMemberStatus::Running,
3347                            timestamp: OffsetDateTime::now_utc(),
3348                        },
3349                    ))
3350                    .await;
3351                }
3352            }
3353            let runtime = Arc::clone(self);
3354            let turn_req = req;
3355            let thread_id_for_task = turn_req.thread_id.clone();
3356            let turn_id_for_task = turn_id.clone();
3357            tokio::spawn(async move {
3358                let result = Abortable::new(
3359                    runtime.run_turn(
3360                        turn_req,
3361                        turn_id_for_task.clone(),
3362                        initial_mailbox_ack,
3363                        steering,
3364                    ),
3365                    abort_registration,
3366                )
3367                .await;
3368                /*
3369                 * A failed sibling in a parallel tool batch drops in-flight external tool
3370                 * futures (`try_join_all` in `route_tool_calls`), stranding their
3371                 * `pending_external_tool_calls` entries. Sweep before reporting the turn
3372                 * outcome so every `thread/toolExecutionRequested` gets a terminal
3373                 * resolution; on clean completion the map holds nothing for this turn.
3374                 */
3375                runtime
3376                    .cancel_pending_external_tool_calls_for_turn(&turn_id_for_task)
3377                    .await;
3378                let completed = matches!(&result, Ok(Ok(TurnRunOutcome::Completed)));
3379                match &result {
3380                    Ok(Err(err)) => {
3381                        let (cleanup, ownership) =
3382                            runtime.await_provider_turn_cleanup(&turn_id_for_task).await;
3383                        // This wrapper owns terminal failure emission for returned errors.
3384                        runtime
3385                            .emit(RoderEvent::TurnFailed(TurnFailed {
3386                                thread_id: thread_id_for_task.clone(),
3387                                turn_id: turn_id_for_task.clone(),
3388                                error: err.to_string(),
3389                                error_kind: err
3390                                    .downcast_ref::<roder_api::provider_error::ProviderFailure>()
3391                                    .map(|failure| failure.kind.retry_cause().to_string()),
3392                                usage: None,
3393                                timestamp: OffsetDateTime::now_utc(),
3394                            }))
3395                            .await;
3396                        runtime
3397                            .record_turn_lifecycle_with_ownership(
3398                                thread_id_for_task.clone(),
3399                                turn_id_for_task.clone(),
3400                                TurnLifecycleState::Failed,
3401                                cleanup,
3402                                Some(TurnLifecycleReason::ProviderFailure),
3403                                ownership,
3404                            )
3405                            .await;
3406                        let _ = runtime
3407                            .complete_team_member_turn_with_result(
3408                                &thread_id_for_task,
3409                                &turn_id_for_task,
3410                                TeamMemberStatus::Failed,
3411                                None,
3412                                Some(err.to_string()),
3413                            )
3414                            .await;
3415                    }
3416                    Ok(Ok(TurnRunOutcome::Stopped)) => {
3417                        let (cleanup, ownership) =
3418                            runtime.await_provider_turn_cleanup(&turn_id_for_task).await;
3419                        runtime
3420                            .record_turn_lifecycle_with_ownership(
3421                                thread_id_for_task.clone(),
3422                                turn_id_for_task.clone(),
3423                                TurnLifecycleState::Failed,
3424                                cleanup,
3425                                Some(TurnLifecycleReason::ProviderFailure),
3426                                ownership,
3427                            )
3428                            .await;
3429                        let _ = runtime
3430                            .complete_team_member_turn_with_result(
3431                                &thread_id_for_task,
3432                                &turn_id_for_task,
3433                                TeamMemberStatus::Failed,
3434                                None,
3435                                Some("turn stopped before completion".to_string()),
3436                            )
3437                            .await;
3438                    }
3439                    Err(_) => {
3440                        let reason = if let Some(handle) = runtime
3441                            .turn_drains
3442                            .read()
3443                            .await
3444                            .get(&turn_id_for_task)
3445                            .cloned()
3446                        {
3447                            handle
3448                                .interrupt_reason
3449                                .lock()
3450                                .await
3451                                .unwrap_or(TurnLifecycleReason::RuntimeFailure)
3452                        } else {
3453                            TurnLifecycleReason::RuntimeFailure
3454                        };
3455                        let (cleanup, ownership) =
3456                            runtime.await_provider_turn_cleanup(&turn_id_for_task).await;
3457                        runtime
3458                            .record_turn_lifecycle_with_ownership(
3459                                thread_id_for_task.clone(),
3460                                turn_id_for_task.clone(),
3461                                TurnLifecycleState::Interrupted,
3462                                cleanup,
3463                                Some(reason),
3464                                ownership,
3465                            )
3466                            .await;
3467                        runtime
3468                            .emit(RoderEvent::TurnInterrupted(TurnInterrupted {
3469                                thread_id: thread_id_for_task.clone(),
3470                                turn_id: turn_id_for_task.clone(),
3471                                timestamp: OffsetDateTime::now_utc(),
3472                            }))
3473                            .await;
3474                        let _ = runtime
3475                            .complete_team_member_turn_with_result(
3476                                &thread_id_for_task,
3477                                &turn_id_for_task,
3478                                TeamMemberStatus::Interrupted,
3479                                None,
3480                                None,
3481                            )
3482                            .await;
3483                    }
3484                    Ok(Ok(TurnRunOutcome::Completed)) => {}
3485                }
3486                runtime
3487                    .teams
3488                    .release_mailbox_reservations_for_turn(&turn_id_for_task)
3489                    .await;
3490                runtime.active_turns.write().await.remove(&turn_id_for_task);
3491                if let Some(drain) = runtime.turn_drains.write().await.remove(&turn_id_for_task) {
3492                    drain.completed.store(true, Ordering::Release);
3493                    drain.completed_notify.notify_waiters();
3494                }
3495                runtime
3496                    .active_turn_selections
3497                    .write()
3498                    .await
3499                    .remove(&turn_id_for_task);
3500                runtime
3501                    .active_turn_contexts
3502                    .write()
3503                    .await
3504                    .remove(&turn_id_for_task);
3505                if !completed {
3506                    // Completion paths above consume registered provider cleanup
3507                    // handles. A setup failure before an engine can stream has no
3508                    // such handle; this is a harmless final defensive sweep.
3509                    let _ = runtime.provider_turn_cleanups.lock().map(|mut cleanups| {
3510                        cleanups.remove(&turn_id_for_task);
3511                    });
3512                }
3513                runtime.active_turns_changed.notify_waiters();
3514                if completed {
3515                    let _ = runtime
3516                        .continue_active_goal_after_turn(thread_id_for_task)
3517                        .await;
3518                }
3519            });
3520            drop(turn_admission);
3521            Ok(turn_id)
3522        })
3523    }
3524
3525    pub(crate) async fn has_active_turn_for_thread(&self, thread_id: &ThreadId) -> bool {
3526        self.active_turns
3527            .read()
3528            .await
3529            .values()
3530            .any(|handle| &handle.thread_id == thread_id)
3531    }
3532
3533    async fn ensure_codex_v2_team_capacity(
3534        &self,
3535        team: &TeamState,
3536        resuming_member_id: &str,
3537        candidate_model: Option<&str>,
3538    ) -> anyhow::Result<()> {
3539        if !is_codex_v2_team(team)
3540            && !candidate_model
3541                .is_some_and(|model| model_supports_reasoning_effort(model, REASONING_ULTRA))
3542        {
3543            return Ok(());
3544        }
3545        let active_thread_ids = self
3546            .active_turns
3547            .read()
3548            .await
3549            .values()
3550            .map(|handle| handle.thread_id.clone())
3551            .collect::<std::collections::HashSet<_>>();
3552        let resident_threads = 1 + team
3553            .members
3554            .iter()
3555            .filter(|member| {
3556                member.role != roder_api::teams::TeamMemberRole::Lead
3557                    && member.id != resuming_member_id
3558                    && active_thread_ids.contains(&member.thread_id)
3559            })
3560            .count();
3561        anyhow::ensure!(
3562            resident_threads < codex_v2::CODEX_V2_MAX_RESIDENT_TEAM_THREADS,
3563            "agent thread limit reached for this Codex V2 team: maximum {} resident threads (the lead plus 3 running subagents)",
3564            codex_v2::CODEX_V2_MAX_RESIDENT_TEAM_THREADS
3565        );
3566        Ok(())
3567    }
3568
3569    /// Number of currently running turns across all threads. Hosted runtime
3570    /// pools use this to avoid evicting tenants with active work.
3571    pub async fn active_turn_count(&self) -> usize {
3572        self.active_turns.read().await.len()
3573    }
3574
3575    pub async fn active_turn_for_thread(&self, thread_id: &ThreadId) -> Option<TurnId> {
3576        self.active_turns
3577            .read()
3578            .await
3579            .iter()
3580            .find_map(|(turn_id, handle)| (&handle.thread_id == thread_id).then(|| turn_id.clone()))
3581    }
3582
3583    pub async fn thread_activity(&self, thread_id: &ThreadId) -> ThreadActivity {
3584        let active_turn_id = self.active_turn_for_thread(thread_id).await;
3585        let draining_turn_id = if active_turn_id.is_none() {
3586            self.turn_drains
3587                .read()
3588                .await
3589                .iter()
3590                .find_map(|(turn_id, drain)| {
3591                    (&drain.thread_id == thread_id).then(|| turn_id.clone())
3592                })
3593        } else {
3594            None
3595        };
3596        let Some(active_turn_id) = active_turn_id.or(draining_turn_id) else {
3597            return ThreadActivity::default();
3598        };
3599
3600        let mut active_flags = Vec::new();
3601        {
3602            let pending_approvals = self.pending_tool_approvals.lock().await;
3603            if pending_approvals
3604                .values()
3605                .any(|pending| &pending.thread_id == thread_id && pending.turn_id == active_turn_id)
3606            {
3607                active_flags.push("approvalRequired".to_string());
3608            }
3609        }
3610        {
3611            let pending_inputs = self.pending_user_inputs.lock().await;
3612            if pending_inputs
3613                .values()
3614                .any(|pending| &pending.thread_id == thread_id && pending.turn_id == active_turn_id)
3615            {
3616                active_flags.push("userInputRequired".to_string());
3617            }
3618        }
3619        {
3620            let pending_external = self.pending_external_tool_calls.lock().await;
3621            if pending_external
3622                .values()
3623                .any(|pending| &pending.thread_id == thread_id && pending.turn_id == active_turn_id)
3624            {
3625                active_flags.push("externalToolPending".to_string());
3626            }
3627        }
3628        let interrupting = self
3629            .active_turns
3630            .read()
3631            .await
3632            .get(&active_turn_id)
3633            .is_some_and(|handle| handle.drain.interrupt_requested.load(Ordering::Acquire))
3634            || self
3635                .turn_drains
3636                .read()
3637                .await
3638                .get(&active_turn_id)
3639                .is_some_and(|drain| drain.interrupt_requested.load(Ordering::Acquire));
3640        if interrupting {
3641            active_flags.push("interrupting".to_string());
3642        }
3643        if self.pending_plan_exit().await.is_some_and(|pending| {
3644            &pending.thread_id == thread_id && pending.turn_id == active_turn_id
3645        }) {
3646            active_flags.push("planExitRequired".to_string());
3647        }
3648
3649        ThreadActivity {
3650            active_turn_id: Some(active_turn_id),
3651            active_flags,
3652        }
3653    }
3654
3655    pub async fn interrupt_turn(&self, thread_id: ThreadId, turn_id: TurnId) -> anyhow::Result<()> {
3656        self.interrupt_turn_with_reason(thread_id, turn_id, TurnLifecycleReason::UserInterrupt)
3657            .await
3658    }
3659
3660    async fn interrupt_turn_with_reason(
3661        &self,
3662        thread_id: ThreadId,
3663        turn_id: TurnId,
3664        reason: TurnLifecycleReason,
3665    ) -> anyhow::Result<()> {
3666        let handle = self.active_turns.write().await.remove(&turn_id);
3667        if let Some(handle) = handle {
3668            if handle
3669                .drain
3670                .interrupt_requested
3671                .swap(true, Ordering::AcqRel)
3672            {
3673                return Ok(());
3674            }
3675            *handle.drain.interrupt_reason.lock().await = Some(reason);
3676            self.turn_drains
3677                .write()
3678                .await
3679                .insert(turn_id.clone(), handle.drain.clone());
3680            let ownership = self.provider_cleanup_ownership(&turn_id);
3681            self.record_turn_lifecycle_with_ownership(
3682                thread_id.clone(),
3683                turn_id.clone(),
3684                TurnLifecycleState::InterruptRequested,
3685                TurnCleanupState::Requested,
3686                Some(reason),
3687                ownership,
3688            )
3689            .await;
3690            handle.abort.abort();
3691            self.active_turns_changed.notify_waiters();
3692        }
3693        self.active_turn_selections.write().await.remove(&turn_id);
3694        self.cancel_pending_external_tool_calls_for_turn(&turn_id)
3695            .await;
3696        Ok(())
3697    }
3698
3699    pub async fn steer_turn(
3700        &self,
3701        thread_id: ThreadId,
3702        turn_id: TurnId,
3703        message: String,
3704        images: Vec<InputImage>,
3705    ) -> anyhow::Result<()> {
3706        self.enqueue_turn_steer(thread_id, turn_id, message, images, None)
3707            .await
3708    }
3709
3710    pub(crate) async fn has_pending_turn_steers(&self, turn_id: &TurnId) -> bool {
3711        let active = self.active_turns.read().await.get(turn_id).cloned();
3712        let Some(active) = active else {
3713            return false;
3714        };
3715        let has_pending = !active.steers.lock().await.is_empty();
3716        has_pending
3717    }
3718
3719    async fn steer_turn_with_mailbox_ack(
3720        &self,
3721        thread_id: ThreadId,
3722        turn_id: TurnId,
3723        message: String,
3724        message_ids: Vec<String>,
3725        team_id: TeamId,
3726    ) -> anyhow::Result<()> {
3727        self.enqueue_turn_steer(
3728            thread_id,
3729            turn_id,
3730            message,
3731            Vec::new(),
3732            Some(MailboxDeliveryAck {
3733                team_id,
3734                message_ids,
3735            }),
3736        )
3737        .await
3738    }
3739
3740    async fn deliver_pending_team_mailbox(
3741        &self,
3742        team_id: &str,
3743        member_id: &str,
3744        member_thread_id: ThreadId,
3745        turn_id: TurnId,
3746    ) -> anyhow::Result<()> {
3747        let pending = self
3748            .teams
3749            .reserve_pending_mailbox_messages(team_id, member_id, &turn_id)
3750            .await?;
3751        if pending.is_empty() {
3752            return Ok(());
3753        }
3754        let message_ids = pending
3755            .iter()
3756            .map(|message| message.id.clone())
3757            .collect::<Vec<_>>();
3758        let team = self
3759            .read_team(team_id)
3760            .await
3761            .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
3762        if let Err(error) = self
3763            .steer_turn_with_mailbox_ack(
3764                member_thread_id,
3765                turn_id.clone(),
3766                format_mailbox_messages(&team, &pending),
3767                message_ids.clone(),
3768                team_id.to_string(),
3769            )
3770            .await
3771        {
3772            self.teams
3773                .release_mailbox_reservations(&turn_id, &message_ids)
3774                .await;
3775            return Err(error);
3776        }
3777        Ok(())
3778    }
3779
3780    async fn enqueue_turn_steer(
3781        &self,
3782        thread_id: ThreadId,
3783        turn_id: TurnId,
3784        message: String,
3785        images: Vec<InputImage>,
3786        mailbox_ack: Option<MailboxDeliveryAck>,
3787    ) -> anyhow::Result<()> {
3788        let message = message.trim().to_string();
3789        if message.is_empty() && images.is_empty() {
3790            return Ok(());
3791        }
3792
3793        let Some(active) = self.active_turns.read().await.get(&turn_id).cloned() else {
3794            anyhow::bail!("no active turn to steer");
3795        };
3796        anyhow::ensure!(
3797            active.thread_id == thread_id,
3798            "turn does not belong to thread"
3799        );
3800        {
3801            let mut steers = active.steers.lock().await;
3802            steers.push(QueuedTurnSteer {
3803                message: UserMessage::with_images(message.clone(), images),
3804                mailbox_ack,
3805            });
3806            active
3807                .steer_changed
3808                .send_modify(|generation| *generation = generation.wrapping_add(1));
3809        }
3810        self.emit(RoderEvent::TurnSteered(TurnSteered {
3811            thread_id,
3812            turn_id,
3813            message,
3814            timestamp: OffsetDateTime::now_utc(),
3815        }))
3816        .await;
3817        Ok(())
3818    }
3819
3820    pub async fn tool_specs(&self) -> Vec<roder_api::tools::ToolSpec> {
3821        let cfg = self.config.read().await;
3822        let model_profile =
3823            model_profile_for_provider_model(&cfg, &cfg.default_provider, &cfg.default_model);
3824        self.filtered_tool_specs(
3825            &cfg,
3826            &cfg.default_model,
3827            &cfg.default_provider,
3828            model_profile.as_ref(),
3829            &[],
3830            &[],
3831        )
3832    }
3833
3834    pub fn subagent_definitions(&self) -> Vec<SubagentDefinition> {
3835        self.registry
3836            .subagent_dispatchers
3837            .iter()
3838            .flat_map(|dispatcher| dispatcher.definitions())
3839            .collect()
3840    }
3841
3842    async fn run_turn(
3843        self: &Arc<Self>,
3844        req: StartTurnRequest,
3845        turn_id: TurnId,
3846        initial_mailbox_ack: Option<MailboxDeliveryAck>,
3847        mut steering: tokio::sync::watch::Receiver<u64>,
3848    ) -> anyhow::Result<TurnRunOutcome> {
3849        self.ensure_execution_authority()?;
3850        let turn_started_at = OffsetDateTime::now_utc();
3851        self.emit(RoderEvent::TurnStarted(TurnStarted {
3852            thread_id: req.thread_id.clone(),
3853            turn_id: turn_id.clone(),
3854            runtime_profile: self.config.read().await.runtime_profile,
3855            timestamp: turn_started_at,
3856        }))
3857        .await;
3858        self.persist_turn_item(
3859            &req.thread_id,
3860            &turn_id,
3861            &TranscriptItem::UserMessage(UserMessage::with_images(
3862                req.message.clone(),
3863                req.images.clone(),
3864            )),
3865        )
3866        .await?;
3867        crate::hooks::run_lifecycle(
3868            self,
3869            &req.thread_id,
3870            &turn_id,
3871            Some(&req.workspace),
3872            "UserPromptSubmit",
3873            None,
3874            serde_json::json!({"prompt": req.message}),
3875        )
3876        .await;
3877        if let Some(ack) = initial_mailbox_ack {
3878            self.teams
3879                .mark_mailbox_messages_delivered(&ack.team_id, &turn_id, &ack.message_ids)
3880                .await?;
3881        }
3882
3883        let mut cfg = self.config.read().await.clone();
3884        let runtime_profile = cfg.runtime_profile;
3885        let turn_deadline = turn_deadline_for_config(&cfg);
3886        let deadline_finalization_reserve =
3887            crate::deadline_policy::finalization_reserve_seconds(cfg.turn_deadline_seconds);
3888        let selection_mode = self.selection_mode_for_thread(&req.thread_id).await?;
3889        let concrete_selection = selection_mode
3890            .as_ref()
3891            .map(ModelSelectionMode::concrete_selection);
3892        let default_provider = req
3893            .provider_override
3894            .clone()
3895            .or_else(|| {
3896                concrete_selection
3897                    .as_ref()
3898                    .map(|selection| selection.provider.clone())
3899            })
3900            .unwrap_or(cfg.default_provider.clone());
3901        let default_model = req
3902            .model_override
3903            .clone()
3904            .or_else(|| {
3905                concrete_selection
3906                    .as_ref()
3907                    .map(|selection| selection.model.clone())
3908            })
3909            .unwrap_or(cfg.default_model.clone());
3910        if let Some(reasoning) = req.reasoning_override.as_deref().or_else(|| {
3911            selection_mode
3912                .as_ref()
3913                .and_then(ModelSelectionMode::reasoning)
3914        }) {
3915            validate_reasoning_effort(&default_model, reasoning)?;
3916            cfg.reasoning = Some(reasoning.to_string());
3917        }
3918        let turn_has_concrete_model_override =
3919            req.provider_override.is_some() || req.model_override.is_some();
3920        let (turn_inference_router, turn_inference_router_profile) = match &selection_mode {
3921            Some(ModelSelectionMode::Auto {
3922                router_id, profile, ..
3923            }) if !turn_has_concrete_model_override => (
3924                RuntimeInferenceRouterConfig {
3925                    enabled: true,
3926                    router_id: Some(router_id.clone()),
3927                },
3928                profile.clone(),
3929            ),
3930            _ => (RuntimeInferenceRouterConfig::disabled(), None),
3931        };
3932        let mut provider = default_provider.clone();
3933        let mut model = default_model.clone();
3934        let mut model_profile = model_profile_for_provider_model(&cfg, &provider, &model);
3935        let workspace = req.workspace.clone();
3936        let compaction_generation_at_start = self.compaction_generation(&req.thread_id);
3937        let mut transcript = self.transcript_for_turn(&req, &turn_id, &model).await?;
3938        let mut compacted_this_turn =
3939            self.compaction_generation(&req.thread_id) != compaction_generation_at_start;
3940        let effective_policy_mode = self.effective_policy_mode_for_thread(&req.thread_id).await;
3941        let agent_swarm_mode_active = self
3942            .effective_agent_swarm_mode_for_thread(&req.thread_id)
3943            .await;
3944        let ultra_mode_active = self.effective_ultra_mode_for_thread(&req.thread_id).await;
3945        let thread_overrides = self.thread_turn_overrides(&req.thread_id).await?;
3946        let mut final_assistant_text = String::new();
3947        let mut final_provider_metadata = None;
3948        let mut exhausted_tool_rounds = true;
3949        let mut verification_gate =
3950            VerificationGateState::new(req.message.clone(), runtime_profile);
3951        let mut speed_policy = SpeedPolicyState::default();
3952        let mut reliability = TurnReliabilityState::default();
3953        let mut turn_usage = TokenUsage::default();
3954        // Overwritten on every inference step's Completed event so only the
3955        // terminal step's stop reason survives (mid-turn tool_use steps must
3956        // not leak onto turn/completed).
3957        let mut turn_finish_reason: Option<String> = None;
3958        let mut deadline_finalization_requested = false;
3959        let mut deadline_scoreable_completion_requested = false;
3960        let mut task_ledger_completion_reminders = 0_u8;
3961        let mut task_ledger_scoreable_checkpoints = 0_u8;
3962        let mut empty_tool_call_nudges_used = 0_u32;
3963        let mut provider_stream_retry_attempts = 0_u32;
3964        let mut context_limit_recovery_attempts = 0_u32;
3965        let mut routing_candidates = None;
3966        let routing_transcript_start = transcript.len().saturating_sub(1);
3967        let mut routing_escalations = 0_u32;
3968        let mut model_switch_summary_selection = None::<ModelSelection>;
3969
3970        'tool_rounds: for round_index in 0..MAX_TOOL_ROUNDS_PER_TURN {
3971            if let Some(deadline) = turn_deadline
3972                && deadline_expired(deadline)
3973            {
3974                self.fail_turn_due_to_deadline(&req.thread_id, &turn_id, deadline, &transcript)
3975                    .await?;
3976                return Ok(TurnRunOutcome::Stopped);
3977            }
3978            let steers = self.drain_turn_steers(&turn_id, &mut steering).await;
3979            self.append_steers(&req, &turn_id, &mut transcript, steers)
3980                .await?;
3981            if runtime_profile == RuntimeProfile::Eval
3982                && let Some(remaining) = crate::deadline_policy::should_start_finalization(
3983                    turn_deadline,
3984                    deadline_finalization_reserve,
3985                    deadline_finalization_requested || deadline_scoreable_completion_requested,
3986                )
3987            {
3988                if req.task_ledger_required
3989                    && task_ledger_completion_reminders < TASK_LEDGER_COMPLETION_REMINDER_LIMIT
3990                    && let Some(prompt) = task_ledger_completion_prompt(&transcript)
3991                {
3992                    task_ledger_completion_reminders += 1;
3993                    deadline_scoreable_completion_requested = true;
3994                    let item = TranscriptItem::UserMessage(UserMessage::text(
3995                        task_ledger_deadline_completion_prompt(
3996                            remaining,
3997                            deadline_finalization_reserve,
3998                            &prompt,
3999                        ),
4000                    ));
4001                    self.persist_turn_item(&req.thread_id, &turn_id, &item)
4002                        .await?;
4003                    transcript.push(item);
4004                    continue 'tool_rounds;
4005                } else {
4006                    self.start_deadline_finalization(
4007                        &req.thread_id,
4008                        &turn_id,
4009                        &mut transcript,
4010                        remaining,
4011                    )
4012                    .await?;
4013                    deadline_finalization_requested = true;
4014                }
4015            }
4016            if runtime_profile == RuntimeProfile::Eval
4017                && req.task_ledger_required
4018                && !deadline_finalization_requested
4019                && task_ledger_scoreable_checkpoints < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
4020                && let Some(remaining) = deadline_remaining_seconds(turn_deadline)
4021                && remaining <= TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS
4022                && remaining > deadline_finalization_reserve
4023                && let Some(prompt) = task_ledger_completion_prompt(&transcript)
4024            {
4025                task_ledger_scoreable_checkpoints += 1;
4026                let item = TranscriptItem::UserMessage(UserMessage::text(
4027                    task_ledger_scoreable_checkpoint_prompt(remaining, &prompt),
4028                ));
4029                self.persist_turn_item(&req.thread_id, &turn_id, &item)
4030                    .await?;
4031                transcript.push(item);
4032                continue 'tool_rounds;
4033            }
4034            if turn_inference_router.is_active() && routing_candidates.is_none() {
4035                routing_candidates =
4036                    Some(collect_inference_routing_candidates(&self.registry).await);
4037            }
4038            let routing_tools_model = model.clone();
4039            let routing_tools_provider = provider.clone();
4040            let routing_tools = self.filtered_tool_specs(
4041                &cfg,
4042                &model,
4043                &provider,
4044                model_profile.as_ref(),
4045                &thread_overrides.tool_allowlist,
4046                &thread_overrides.external_tools,
4047            );
4048            let prior_failures =
4049                transcript_failure_count_since(&transcript, routing_transcript_start)
4050                    .max(reliability.tool_failure_count())
4051                    .saturating_add(provider_stream_retry_attempts);
4052            let routing_selection = route_inference_selection(
4053                &self.registry,
4054                &turn_inference_router,
4055                InferenceRoutingRequest {
4056                    thread_id: &req.thread_id,
4057                    turn_id: &turn_id,
4058                    round_index: round_index as u32,
4059                    runtime_profile,
4060                    phase: speed_policy.phase(),
4061                    profile: turn_inference_router_profile.as_deref(),
4062                    default_selection: ModelSelection {
4063                        provider: default_provider.clone(),
4064                        model: default_model.clone(),
4065                    },
4066                    transcript: &transcript,
4067                    tools: &routing_tools,
4068                    candidates: routing_candidates.as_deref(),
4069                    prior_failures,
4070                    prior_escalations: routing_escalations,
4071                },
4072            )
4073            .await;
4074            if let Some(decision) = routing_selection.decision.clone() {
4075                if matches!(decision.outcome, InferenceRoutingOutcome::Escalated) {
4076                    routing_escalations = routing_escalations.saturating_add(1);
4077                }
4078                self.emit(RoderEvent::InferenceRoutingDecision(
4079                    InferenceRoutingDecisionEvent {
4080                        thread_id: req.thread_id.clone(),
4081                        turn_id: turn_id.clone(),
4082                        round_index: round_index as u32,
4083                        default_selection: ModelSelection {
4084                            provider: default_provider.clone(),
4085                            model: default_model.clone(),
4086                        },
4087                        selected_selection: routing_selection.selection.clone(),
4088                        decision,
4089                        timestamp: OffsetDateTime::now_utc(),
4090                    },
4091                ))
4092                .await;
4093            }
4094            provider = routing_selection.selection.provider.clone();
4095            model = routing_selection.selection.model.clone();
4096            let engine = self.engine_for(&provider)?;
4097            let capabilities = engine.capabilities();
4098            model_profile = model_profile_for_provider_model(&cfg, &provider, &model);
4099            let tools = if capabilities.tool_calls {
4100                if model == routing_tools_model && provider == routing_tools_provider {
4101                    routing_tools.clone()
4102                } else {
4103                    self.filtered_tool_specs(
4104                        &cfg,
4105                        &model,
4106                        &provider,
4107                        model_profile.as_ref(),
4108                        &thread_overrides.tool_allowlist,
4109                        &thread_overrides.external_tools,
4110                    )
4111                }
4112            } else {
4113                Vec::new()
4114            };
4115            let parallel_tool_calls = parallel_tool_calls_for_model(&cfg, &model);
4116            let tool_choice = if tools.is_empty() {
4117                ToolChoice::None
4118            } else {
4119                ToolChoice::Auto
4120            };
4121            let summary_selection = ModelSelection {
4122                provider: provider.clone(),
4123                model: model.clone(),
4124            };
4125            if model_switch_summary_selection.as_ref() != Some(&summary_selection) {
4126                if let Some(summary) = model_switch_summary(
4127                    &transcript,
4128                    model_profile.as_ref(),
4129                    &provider,
4130                    &model,
4131                    &tools,
4132                ) {
4133                    let item = TranscriptItem::UserMessage(UserMessage::text(summary));
4134                    self.persist_turn_item(&req.thread_id, &turn_id, &item)
4135                        .await?;
4136                    transcript.push(item);
4137                }
4138                model_switch_summary_selection = Some(summary_selection);
4139            }
4140
4141            if !capabilities.image_input && transcript_has_images(&transcript) {
4142                self.fail_turn_with_error(
4143                    &req.thread_id,
4144                    &turn_id,
4145                    format!("provider {provider} does not support image input"),
4146                )
4147                .await?;
4148                return Ok(TurnRunOutcome::Stopped);
4149            }
4150            let compaction_generation_before = self.compaction_generation(&req.thread_id);
4151            transcript = self
4152                .compact_transcript_if_needed(
4153                    &req.thread_id,
4154                    &turn_id,
4155                    &provider,
4156                    &model,
4157                    transcript,
4158                    self.compaction_options_for_turn(&req.thread_id, !compacted_this_turn),
4159                )
4160                .await?;
4161            compacted_this_turn |=
4162                self.compaction_generation(&req.thread_id) != compaction_generation_before;
4163
4164            let speed_policy_decision =
4165                speed_policy.decision(runtime_profile, &model, &cfg.speed_policy);
4166            let request_reasoning = reasoning_from_decision(
4167                speed_policy_decision.as_ref(),
4168                routing_selection
4169                    .reasoning
4170                    .clone()
4171                    .unwrap_or_else(|| reasoning_for_model(&cfg, &model)),
4172            );
4173            self.active_turn_selections.write().await.insert(
4174                turn_id.clone(),
4175                ModelSelectionMode::manual(
4176                    provider.clone(),
4177                    model.clone(),
4178                    request_reasoning.level.clone(),
4179                ),
4180            );
4181            if let Some(limit) = reliability.record_model_call(
4182                &cfg.reliability,
4183                runtime_profile == RuntimeProfile::Interactive,
4184            ) {
4185                self.fail_turn_due_to_reliability_limit(
4186                    &req.thread_id,
4187                    &turn_id,
4188                    &provider,
4189                    &model,
4190                    limit,
4191                    &transcript,
4192                )
4193                .await?;
4194                return Ok(TurnRunOutcome::Stopped);
4195            }
4196            self.emit(RoderEvent::InferenceStarted(InferenceStarted {
4197                thread_id: req.thread_id.clone(),
4198                turn_id: turn_id.clone(),
4199                engine_id: engine.id(),
4200                model: ModelSelection {
4201                    provider: provider.clone(),
4202                    model: model.clone(),
4203                },
4204                reasoning: request_reasoning.clone(),
4205                speed_policy: speed_policy_decision.clone(),
4206                deadline_remaining_seconds: deadline_remaining_seconds(turn_deadline),
4207                timestamp: OffsetDateTime::now_utc(),
4208            }))
4209            .await;
4210
4211            let mut instructions = req.instructions.clone();
4212            if let Some(extra) = &thread_overrides.developer_instructions {
4213                instructions = apply_thread_developer_instructions(instructions, extra);
4214            }
4215            if let Some(context) = req.developer_context.as_deref() {
4216                instructions = apply_turn_developer_context(instructions, context);
4217            }
4218            let mut instructions = apply_runtime_profile(instructions, runtime_profile);
4219            if let Some(profile) = &model_profile {
4220                instructions = apply_model_instruction_overlay(instructions, profile);
4221            }
4222            if req.task_ledger_required
4223                && runtime_profile == RuntimeProfile::Eval
4224                && !transcript_has_task_ledger(&transcript)
4225            {
4226                instructions = apply_task_ledger_required(instructions);
4227            }
4228            if effective_policy_mode == PolicyMode::Plan {
4229                instructions = apply_plan_mode(instructions);
4230            }
4231            if agent_swarm_mode_active {
4232                instructions = apply_agent_swarm_mode(instructions);
4233            }
4234            // Ultra mode (any model) enables proactive multi-agent policy.
4235            // Codex Sol/Terra Ultra effort still does too without requiring the
4236            // mode flag. Models that advertise Ultra effort keep the
4237            // explicit-request-only policy on lower efforts when ultra mode is
4238            // off; other models get multi-agent policy only when ultra mode is on.
4239            let model_has_ultra_effort = model_supports_reasoning_effort(&model, REASONING_ULTRA);
4240            let effort_is_ultra = request_reasoning.level.as_deref() == Some(REASONING_ULTRA);
4241            let proactive_multi_agent = ultra_mode_active || effort_is_ultra;
4242            if ultra_mode_active || model_has_ultra_effort {
4243                instructions = apply_codex_multi_agent_mode(instructions, proactive_multi_agent);
4244            }
4245            instructions = self
4246                .goals
4247                .apply_goal_instructions(&req.thread_id, instructions)
4248                .await?;
4249            instructions =
4250                apply_parallel_web_tools(instructions, tools.iter().map(|spec| spec.name.as_str()));
4251            let mut request_metadata = serde_json::json!({});
4252            if let Some(decision) = &speed_policy_decision {
4253                request_metadata["speedPolicy"] = serde_json::json!(decision);
4254            }
4255            if let Some(decision) = routing_selection.decision.as_ref() {
4256                request_metadata["inferenceRouting"] = serde_json::json!(decision);
4257            }
4258            if let Some(remaining) = deadline_remaining_seconds(turn_deadline) {
4259                request_metadata["deadlineRemainingSeconds"] = serde_json::json!(remaining);
4260            }
4261            if let Some(profile) = &model_profile {
4262                request_metadata["modelProfile"] = serde_json::json!({
4263                    "model": profile.model,
4264                    "providerFamily": profile.provider_family,
4265                    "editTool": profile.edit_tool,
4266                    "schemaPolicy": profile.schema_policy,
4267                    "instructionOverlay": profile.instruction_overlay,
4268                    "parallelToolCalls": profile.parallel_tool_calls,
4269                    "autoCompactTokenLimit": profile.auto_compact_token_limit,
4270                });
4271            }
4272            let task_ledger_required_this_round = req.task_ledger_required
4273                && runtime_profile == RuntimeProfile::Eval
4274                && !deadline_finalization_requested
4275                && !transcript_has_task_ledger(&transcript);
4276            let task_ledger_tools = (capabilities.tool_calls && task_ledger_required_this_round)
4277                .then(|| self.task_ledger_tool_specs(model_profile.as_ref()))
4278                .filter(|tools| !tools.is_empty());
4279            let request_tools = if deadline_finalization_requested {
4280                Vec::new()
4281            } else if let Some(ledger_tools) = &task_ledger_tools {
4282                ledger_tools.clone()
4283            } else {
4284                tools.clone()
4285            };
4286            let request_tool_choice = if deadline_finalization_requested {
4287                ToolChoice::None
4288            } else if task_ledger_tools.is_some() {
4289                ToolChoice::Specific(TASK_LEDGER_TOOL_NAME.to_string())
4290            } else {
4291                tool_choice.clone()
4292            };
4293            if deadline_finalization_requested {
4294                request_metadata["deadlineFinalization"] = serde_json::json!({
4295                    "reserveSeconds": deadline_finalization_reserve,
4296                    "remainingSeconds": deadline_remaining_seconds(turn_deadline),
4297                });
4298            } else if deadline_scoreable_completion_requested {
4299                request_metadata["deadlineScoreableCompletion"] = serde_json::json!({
4300                    "reserveSeconds": deadline_finalization_reserve,
4301                    "remainingSeconds": deadline_remaining_seconds(turn_deadline),
4302                });
4303            }
4304            let mut eager_tools = EagerTools::new(
4305                self,
4306                &request_tools,
4307                &thread_overrides.external_tools,
4308                parallel_tool_calls,
4309            );
4310            let mut request_reliability: ReliabilityRequestPolicy = cfg.reliability.clone().into();
4311            request_reliability.provider_retry_max_attempts = request_reliability
4312                .provider_retry_max_attempts
4313                .saturating_sub(provider_stream_retry_attempts)
4314                .max(1);
4315            let request = AgentInferenceRequest {
4316                model: ModelSelection {
4317                    provider: provider.clone(),
4318                    model: model.clone(),
4319                },
4320                instructions,
4321                transcript: transcript.clone(),
4322                tools: request_tools,
4323                tool_choice: request_tool_choice,
4324                reasoning: request_reasoning,
4325                output: OutputConfig::default(),
4326                runtime: RuntimeHints {
4327                    auto_compact_token_limit: (provider != roder_api::catalog::PROVIDER_CODEX)
4328                        .then(|| server_side_compaction_threshold(&cfg, &model))
4329                        .flatten(),
4330                    profile: runtime_profile,
4331                    parallel_tool_calls: Some(parallel_tool_calls),
4332                    prompt_cache_key: Some(req.thread_id.clone()),
4333                    hosted_web_search: cfg.hosted_web_search.clone(),
4334                    tool_search: tool_search_for_provider_model(&cfg, &provider, &model),
4335                    speed_policy: speed_policy_decision,
4336                    reliability: Some(request_reliability),
4337                    deadline_remaining_seconds: deadline_remaining_seconds(turn_deadline),
4338                    service_tier: req.service_tier_override.clone(),
4339                    ..RuntimeHints::default()
4340                },
4341                metadata: request_metadata,
4342            };
4343
4344            let mut compaction_template = request.clone();
4345            compaction_template.transcript.clear();
4346            self.compaction_templates
4347                .write()
4348                .await
4349                .insert(req.thread_id.clone(), compaction_template);
4350
4351            let ctx = InferenceTurnContext {
4352                thread_id: &req.thread_id,
4353                turn_id: &turn_id,
4354                tool_executor: Some(std::sync::Arc::new(
4355                    crate::tool_execution::RuntimeTurnToolExecutor {
4356                        runtime: Arc::clone(self),
4357                        thread_id: req.thread_id.clone(),
4358                        turn_id: turn_id.clone(),
4359                        workspace: Some(workspace.clone()),
4360                        deadline: turn_deadline,
4361                    },
4362                )),
4363            };
4364            let stream_future = preemptible(engine.stream_turn(ctx, request), &mut steering);
4365            let mut stream = if let Some((deadline, timeout_action)) = inference_timeout_deadline(
4366                turn_deadline,
4367                runtime_profile,
4368                req.task_ledger_required,
4369                deadline_finalization_reserve,
4370                deadline_finalization_requested || deadline_scoreable_completion_requested,
4371                task_ledger_scoreable_checkpoints,
4372                &transcript,
4373            ) {
4374                match tokio::time::timeout_at(deadline_instant(deadline), stream_future).await {
4375                    Ok(stream) => match stream {
4376                        Ok(stream) => stream,
4377                        Err(err) => {
4378                            if err.is::<SamplingPreempted>() {
4379                                self.finish_sampling_cleanup(&turn_id).await?;
4380                                provider_stream_retry_attempts = 0;
4381                                continue 'tool_rounds;
4382                            }
4383                            self.finish_sampling_cleanup(&turn_id).await?;
4384                            if !deadline_finalization_requested
4385                                && self
4386                                    .retry_sampling_failure(
4387                                        SamplingRetry {
4388                                            thread_id: &req.thread_id,
4389                                            turn_id: &turn_id,
4390                                            provider: &provider,
4391                                            model: &model,
4392                                            config: &cfg.reliability,
4393                                            error: &err,
4394                                        },
4395                                        &mut provider_stream_retry_attempts,
4396                                        &mut steering,
4397                                    )
4398                                    .await
4399                            {
4400                                continue 'tool_rounds;
4401                            }
4402                            let error = err.to_string();
4403                            if self
4404                                .try_recover_from_context_limit(
4405                                    &req.thread_id,
4406                                    &turn_id,
4407                                    &provider,
4408                                    &model,
4409                                    &error,
4410                                    &mut transcript,
4411                                    &mut compacted_this_turn,
4412                                    &mut context_limit_recovery_attempts,
4413                                )
4414                                .await?
4415                            {
4416                                continue 'tool_rounds;
4417                            }
4418                            return Err(err);
4419                        }
4420                    },
4421                    Err(_) => {
4422                        if runtime_profile == RuntimeProfile::Eval
4423                            && !deadline_finalization_requested
4424                        {
4425                            let remaining = deadline_remaining_seconds(turn_deadline).unwrap_or(0);
4426                            if timeout_action == InferenceTimeoutAction::ScoreableCheckpoint
4427                                && task_ledger_scoreable_checkpoints
4428                                    < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
4429                                && let Some(prompt) = task_ledger_completion_prompt(&transcript)
4430                            {
4431                                task_ledger_scoreable_checkpoints += 1;
4432                                let item = TranscriptItem::UserMessage(UserMessage::text(
4433                                    task_ledger_scoreable_checkpoint_prompt(remaining, &prompt),
4434                                ));
4435                                self.persist_turn_item(&req.thread_id, &turn_id, &item)
4436                                    .await?;
4437                                transcript.push(item);
4438                                continue 'tool_rounds;
4439                            }
4440                            self.start_deadline_finalization(
4441                                &req.thread_id,
4442                                &turn_id,
4443                                &mut transcript,
4444                                remaining,
4445                            )
4446                            .await?;
4447                            deadline_finalization_requested = true;
4448                            continue 'tool_rounds;
4449                        }
4450                        self.fail_turn_due_to_deadline(
4451                            &req.thread_id,
4452                            &turn_id,
4453                            deadline,
4454                            &transcript,
4455                        )
4456                        .await?;
4457                        return Ok(TurnRunOutcome::Stopped);
4458                    }
4459                }
4460            } else {
4461                match stream_future.await {
4462                    Ok(stream) => stream,
4463                    Err(err) => {
4464                        if err.is::<SamplingPreempted>() {
4465                            self.finish_sampling_cleanup(&turn_id).await?;
4466                            provider_stream_retry_attempts = 0;
4467                            continue 'tool_rounds;
4468                        }
4469                        self.finish_sampling_cleanup(&turn_id).await?;
4470                        if !deadline_finalization_requested
4471                            && self
4472                                .retry_sampling_failure(
4473                                    SamplingRetry {
4474                                        thread_id: &req.thread_id,
4475                                        turn_id: &turn_id,
4476                                        provider: &provider,
4477                                        model: &model,
4478                                        config: &cfg.reliability,
4479                                        error: &err,
4480                                    },
4481                                    &mut provider_stream_retry_attempts,
4482                                    &mut steering,
4483                                )
4484                                .await
4485                        {
4486                            continue 'tool_rounds;
4487                        }
4488                        let error = err.to_string();
4489                        if self
4490                            .try_recover_from_context_limit(
4491                                &req.thread_id,
4492                                &turn_id,
4493                                &provider,
4494                                &model,
4495                                &error,
4496                                &mut transcript,
4497                                &mut compacted_this_turn,
4498                                &mut context_limit_recovery_attempts,
4499                            )
4500                            .await?
4501                        {
4502                            continue 'tool_rounds;
4503                        }
4504                        return Err(err);
4505                    }
4506                }
4507            };
4508            let mut sampling_output = SamplingOutput::default();
4509            let output_context = OutputContext {
4510                thread_id: &req.thread_id,
4511                turn_id: &turn_id,
4512                profile: model_profile.as_ref(),
4513                provider: &provider,
4514                model: &model,
4515            };
4516            let mut assistant_text = String::new();
4517            let mut phase_messages = Vec::<AssistantMessage>::new();
4518            let mut reasoning_text = String::new();
4519            let mut replayed_tool_call = false;
4520            let mut tool_calls = Vec::new();
4521            let mut patch_progress = PatchProgressTracker::default();
4522            let mut provider_metadata = None;
4523            let mut provider_requests_continuation = false;
4524            // Captured from mid-stream Compaction(completed) events. Codex/OpenAI
4525            // sometimes abort the SSE stream after emitting the compaction item
4526            // ("error decoding response body") before response.completed, so we
4527            // must not rely solely on ProviderMetadata for the boundary.
4528            let mut stream_compaction_item: Option<serde_json::Value> = None;
4529
4530            loop {
4531                let next_future = async {
4532                    loop {
4533                        tokio::select! {
4534                            biased;
4535                            _ = wait_for_steer(&mut steering) => break Some(Err(SamplingPreempted.into())),
4536                            result = eager_tools.poll_result(), if eager_tools.pending() => {
4537                                if let Err(error) = result { break Some(Err(error)); }
4538                            }
4539                            next = stream.next() => break next,
4540                        }
4541                    }
4542                };
4543                let next = if let Some((deadline, timeout_action)) = inference_timeout_deadline(
4544                    turn_deadline,
4545                    runtime_profile,
4546                    req.task_ledger_required,
4547                    deadline_finalization_reserve,
4548                    deadline_finalization_requested || deadline_scoreable_completion_requested,
4549                    task_ledger_scoreable_checkpoints,
4550                    &transcript,
4551                ) {
4552                    match tokio::time::timeout_at(deadline_instant(deadline), next_future).await {
4553                        Ok(next) => next,
4554                        Err(_) => {
4555                            if runtime_profile == RuntimeProfile::Eval
4556                                && !deadline_finalization_requested
4557                            {
4558                                let remaining =
4559                                    deadline_remaining_seconds(turn_deadline).unwrap_or(0);
4560                                if timeout_action == InferenceTimeoutAction::ScoreableCheckpoint
4561                                    && task_ledger_scoreable_checkpoints
4562                                        < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
4563                                    && let Some(prompt) = task_ledger_completion_prompt(&transcript)
4564                                {
4565                                    task_ledger_scoreable_checkpoints += 1;
4566                                    let item = TranscriptItem::UserMessage(UserMessage::text(
4567                                        task_ledger_scoreable_checkpoint_prompt(remaining, &prompt),
4568                                    ));
4569                                    self.persist_turn_item(&req.thread_id, &turn_id, &item)
4570                                        .await?;
4571                                    transcript.push(item);
4572                                    continue 'tool_rounds;
4573                                }
4574                                self.start_deadline_finalization(
4575                                    &req.thread_id,
4576                                    &turn_id,
4577                                    &mut transcript,
4578                                    remaining,
4579                                )
4580                                .await?;
4581                                deadline_finalization_requested = true;
4582                                continue 'tool_rounds;
4583                            }
4584                            self.fail_turn_due_to_deadline(
4585                                &req.thread_id,
4586                                &turn_id,
4587                                deadline,
4588                                &transcript,
4589                            )
4590                            .await?;
4591                            return Ok(TurnRunOutcome::Stopped);
4592                        }
4593                    }
4594                } else {
4595                    next_future.await
4596                };
4597                let res = next.unwrap_or_else(|| {
4598                    Err(roder_api::provider_error::ProviderFailure::new(
4599                        roder_api::provider_error::ProviderFailureKind::StreamInterrupted,
4600                        "provider stream ended before terminal completion",
4601                    )
4602                    .into())
4603                });
4604                let event = match res {
4605                    Ok(event) => event,
4606                    Err(err) => {
4607                        drop(stream);
4608                        eager_tools.drain().await?;
4609                        for result in eager_tools.take_results(&tool_calls) {
4610                            verification_gate.record_tool_result(&result);
4611                            transcript.push(TranscriptItem::ToolResult(result));
4612                        }
4613                        self.finish_sampling_cleanup(&turn_id).await?;
4614                        if err.is::<SamplingPreempted>() {
4615                            provider_stream_retry_attempts = 0;
4616                            continue 'tool_rounds;
4617                        }
4618                        if !deadline_finalization_requested
4619                            && self
4620                                .retry_sampling_failure(
4621                                    SamplingRetry {
4622                                        thread_id: &req.thread_id,
4623                                        turn_id: &turn_id,
4624                                        provider: &provider,
4625                                        model: &model,
4626                                        config: &cfg.reliability,
4627                                        error: &err,
4628                                    },
4629                                    &mut provider_stream_retry_attempts,
4630                                    &mut steering,
4631                                )
4632                                .await
4633                        {
4634                            continue 'tool_rounds;
4635                        }
4636                        let error = err.to_string();
4637                        if self
4638                            .try_recover_from_context_limit(
4639                                &req.thread_id,
4640                                &turn_id,
4641                                &provider,
4642                                &model,
4643                                &error,
4644                                &mut transcript,
4645                                &mut compacted_this_turn,
4646                                &mut context_limit_recovery_attempts,
4647                            )
4648                            .await?
4649                        {
4650                            continue 'tool_rounds;
4651                        }
4652                        return Err(err);
4653                    }
4654                };
4655
4656                let inference_timestamp = OffsetDateTime::now_utc();
4657                self.emit(RoderEvent::InferenceEventReceived(InferenceEventReceived {
4658                    thread_id: req.thread_id.clone(),
4659                    turn_id: turn_id.clone(),
4660                    event: event.clone(),
4661                    timestamp: inference_timestamp,
4662                }))
4663                .await;
4664
4665                match event {
4666                    InferenceEvent::MessageDelta(delta) => {
4667                        if let Some((team_id, member)) =
4668                            self.teams.member_for_thread(&req.thread_id).await
4669                        {
4670                            self.emit(RoderEvent::TeamMemberMessageDelta(TeamMemberMessageDelta {
4671                                team_id,
4672                                member_id: member.id,
4673                                member_thread_id: req.thread_id.clone(),
4674                                turn_id: turn_id.clone(),
4675                                delta: delta.text.clone(),
4676                                timestamp: OffsetDateTime::now_utc(),
4677                            }))
4678                            .await;
4679                        }
4680                        if is_final_answer_phase(delta.phase.as_deref()) {
4681                            assistant_text.push_str(&delta.text);
4682                        } else if let Some(last) = phase_messages.last_mut()
4683                            && last.phase == delta.phase
4684                        {
4685                            last.text.push_str(&delta.text);
4686                        } else {
4687                            phase_messages.push(AssistantMessage {
4688                                text: delta.text,
4689                                phase: delta.phase,
4690                            });
4691                        }
4692                    }
4693                    InferenceEvent::ReasoningDelta(delta) => reasoning_text.push_str(&delta.text),
4694                    InferenceEvent::ToolCallCompleted(call) => {
4695                        if completed_call_replayed(&transcript, &call)? {
4696                            replayed_tool_call = true;
4697                            continue;
4698                        }
4699                        if let Some(progress) = patch_progress.complete(
4700                            &req.thread_id,
4701                            &turn_id,
4702                            &call.id,
4703                            &call.name,
4704                            &call.arguments,
4705                        ) {
4706                            self.emit(RoderEvent::PatchProgress(progress)).await;
4707                        }
4708                        if !tool_calls
4709                            .iter()
4710                            .any(|previous: &ToolCallCompleted| previous.id == call.id)
4711                        {
4712                            eager_tools
4713                                .schedule(
4714                                    self,
4715                                    &output_context,
4716                                    &call,
4717                                    workspace.as_str(),
4718                                    turn_deadline,
4719                                    &mut transcript,
4720                                )
4721                                .await?;
4722                            tool_calls.push(call);
4723                        }
4724                    }
4725                    InferenceEvent::OutputItemCompleted(item) => {
4726                        sampling_output
4727                            .complete_item(self, &output_context, item, &mut transcript)
4728                            .await?;
4729                    }
4730                    InferenceEvent::Failed(failure) => {
4731                        speed_policy.record_failure();
4732                        let error = failure.message;
4733                        if self
4734                            .try_recover_from_context_limit(
4735                                &req.thread_id,
4736                                &turn_id,
4737                                &provider,
4738                                &model,
4739                                &error,
4740                                &mut transcript,
4741                                &mut compacted_this_turn,
4742                                &mut context_limit_recovery_attempts,
4743                            )
4744                            .await?
4745                        {
4746                            continue 'tool_rounds;
4747                        }
4748                        self.persist_turn_item(
4749                            &req.thread_id,
4750                            &turn_id,
4751                            &TranscriptItem::Error(ErrorRecord {
4752                                message: error.clone(),
4753                            }),
4754                        )
4755                        .await?;
4756                        self.emit(RoderEvent::TurnFailed(TurnFailed {
4757                            thread_id: req.thread_id.clone(),
4758                            turn_id: turn_id.clone(),
4759                            error: error.clone(),
4760                            error_kind: None,
4761                            usage: None,
4762                            timestamp: OffsetDateTime::now_utc(),
4763                        }))
4764                        .await;
4765                        self.complete_team_member_turn_with_result(
4766                            &req.thread_id,
4767                            &turn_id,
4768                            TeamMemberStatus::Failed,
4769                            (!assistant_text.trim().is_empty()).then(|| assistant_text.clone()),
4770                            Some(error),
4771                        )
4772                        .await?;
4773                        return Ok(TurnRunOutcome::Stopped);
4774                    }
4775                    InferenceEvent::Usage(usage) => {
4776                        turn_usage.add_assign(&usage);
4777                    }
4778                    InferenceEvent::Completed(metadata) => {
4779                        sampling_output
4780                            .finish(
4781                                self,
4782                                &output_context,
4783                                &assistant_text,
4784                                &phase_messages,
4785                                &reasoning_text,
4786                                &mut transcript,
4787                            )
4788                            .await?;
4789                        provider_stream_retry_attempts = 0;
4790                        turn_finish_reason = metadata
4791                            .stop_reason
4792                            .as_deref()
4793                            .map(finish_reason_from_stop_reason);
4794                        break;
4795                    }
4796                    InferenceEvent::Compaction(progress) => {
4797                        // Persist the boundary as soon as the provider emits a
4798                        // completed compaction item. ChatGPT Codex often aborts
4799                        // the SSE stream right after ("error decoding response
4800                        // body"), so waiting for ProviderMetadata loses the
4801                        // boundary and every subsequent request re-compacts.
4802                        if progress.status == "completed"
4803                            && let Some(item) = progress.item
4804                        {
4805                            let already = stream_compaction_item
4806                                .as_ref()
4807                                .and_then(|existing| {
4808                                    existing.get("id").and_then(serde_json::Value::as_str)
4809                                })
4810                                .zip(item.get("id").and_then(serde_json::Value::as_str))
4811                                .is_some_and(|(a, b)| a == b);
4812                            if !already {
4813                                stream_compaction_item = Some(item.clone());
4814                                let boundary = TranscriptItem::ProviderMetadata(
4815                                    crate::compaction::provider_metadata_from_compaction_item(item),
4816                                );
4817                                self.persist_turn_item(&req.thread_id, &turn_id, &boundary)
4818                                    .await?;
4819                                transcript.push(boundary);
4820                                transcript =
4821                                    crate::compaction::trim_to_last_compaction_boundary(transcript);
4822                                compacted_this_turn = true;
4823                            }
4824                        }
4825                    }
4826                    InferenceEvent::HostedToolCallStarted(_)
4827                    | InferenceEvent::HostedToolCallCompleted(_) => {}
4828                    InferenceEvent::ToolCallStarted(call) => {
4829                        patch_progress.start(&call.id, &call.name)
4830                    }
4831                    InferenceEvent::ToolCallDelta(delta) => {
4832                        if let Some(progress) = patch_progress.delta(
4833                            &req.thread_id,
4834                            &turn_id,
4835                            &delta.id,
4836                            &delta.arguments_delta,
4837                        ) {
4838                            self.emit(RoderEvent::PatchProgress(progress)).await;
4839                        }
4840                    }
4841                    InferenceEvent::ProviderMetadata(mut metadata) => {
4842                        if metadata.get("kind").and_then(serde_json::Value::as_str)
4843                            == Some("reliability_retry_attempt")
4844                        {
4845                            provider_stream_retry_attempts =
4846                                provider_stream_retry_attempts.saturating_add(1);
4847                        }
4848                        provider_requests_continuation |= metadata
4849                            .get("end_turn")
4850                            .and_then(serde_json::Value::as_bool)
4851                            == Some(false);
4852                        if let Some(compaction_item) = stream_compaction_item.as_ref() {
4853                            let _ = crate::compaction::ensure_provider_metadata_compaction(
4854                                &mut metadata,
4855                                compaction_item,
4856                            );
4857                        }
4858                        if metadata.get("output").is_none() {
4859                            let item = TranscriptItem::ProviderMetadata(metadata);
4860                            self.persist_turn_item(&req.thread_id, &turn_id, &item)
4861                                .await?;
4862                            transcript.push(item);
4863                        } else {
4864                            provider_metadata = Some(metadata);
4865                        }
4866                    }
4867                }
4868            }
4869
4870            speed_policy.record_model_output(
4871                !assistant_text.is_empty() || !phase_messages.is_empty(),
4872                tool_calls.len(),
4873            );
4874            if tool_calls.is_empty() {
4875                if provider_requests_continuation || replayed_tool_call {
4876                    if let Some(metadata) = provider_metadata {
4877                        let item = TranscriptItem::ProviderMetadata(metadata);
4878                        self.persist_turn_item(&req.thread_id, &turn_id, &item)
4879                            .await?;
4880                        transcript.push(item);
4881                    }
4882                    continue 'tool_rounds;
4883                }
4884                let steers = self.drain_turn_steers(&turn_id, &mut steering).await;
4885                if !steers.is_empty() {
4886                    if let Some(metadata) = provider_metadata {
4887                        let had_provider_compaction =
4888                            crate::compaction::provider_metadata_has_compaction(&metadata);
4889                        let item = TranscriptItem::ProviderMetadata(metadata);
4890                        self.persist_turn_item(&req.thread_id, &turn_id, &item)
4891                            .await?;
4892                        transcript.push(item);
4893                        if had_provider_compaction {
4894                            // Server-side compaction replaced the prior window;
4895                            // drop pre-boundary items so the next request does
4896                            // not re-send (and re-compact) the full history.
4897                            transcript =
4898                                crate::compaction::trim_to_last_compaction_boundary(transcript);
4899                            compacted_this_turn = true;
4900                        }
4901                    }
4902                    self.append_steers(&req, &turn_id, &mut transcript, steers)
4903                        .await?;
4904                    continue;
4905                }
4906                if !deadline_finalization_requested
4907                    && req.task_ledger_required
4908                    && runtime_profile == RuntimeProfile::Eval
4909                    && task_ledger_completion_reminders < TASK_LEDGER_COMPLETION_REMINDER_LIMIT
4910                    && (!assistant_text.trim().is_empty() || !phase_messages.is_empty())
4911                    && let Some(prompt) = task_ledger_completion_prompt(&transcript)
4912                {
4913                    task_ledger_completion_reminders += 1;
4914                    let item = TranscriptItem::UserMessage(UserMessage::text(prompt));
4915                    self.persist_turn_item(&req.thread_id, &turn_id, &item)
4916                        .await?;
4917                    transcript.push(item);
4918                    continue;
4919                }
4920                if !deadline_finalization_requested
4921                    && let Some(prompt) = verification_gate.blocking_prompt()
4922                {
4923                    speed_policy.record_verification_required();
4924                    self.emit(RoderEvent::VerificationRequired(VerificationRequired {
4925                        thread_id: req.thread_id.clone(),
4926                        turn_id: turn_id.clone(),
4927                        reason: verification_gate.reason(),
4928                        changed_files: verification_gate.changed_files(),
4929                        tool_evidence: verification_gate.tool_evidence.clone(),
4930                        tests_run: verification_gate.tests_run.clone(),
4931                        open_gaps: verification_gate.open_gaps.clone(),
4932                        timestamp: OffsetDateTime::now_utc(),
4933                    }))
4934                    .await;
4935                    let item = TranscriptItem::UserMessage(UserMessage::text(prompt));
4936                    self.persist_turn_item(&req.thread_id, &turn_id, &item)
4937                        .await?;
4938                    transcript.push(item);
4939                    continue;
4940                }
4941                // Loop persistence nudge: in non-interactive/eval runs, when the
4942                // model returns a final message with no tool calls, gently prompt
4943                // it to keep going (bounded by `empty_tool_call_nudges`) before
4944                // ending the turn. Gated to non-interactive/eval so interactive
4945                // chat completions are never delayed, and only fires when the
4946                // model actually produced a final answer to re-examine.
4947                if !deadline_finalization_requested
4948                    && runtime_profile != RuntimeProfile::Interactive
4949                    && cfg.reliability.empty_tool_call_nudges > 0
4950                    && empty_tool_call_nudges_used < cfg.reliability.empty_tool_call_nudges
4951                    && (!assistant_text.trim().is_empty() || !phase_messages.is_empty())
4952                {
4953                    empty_tool_call_nudges_used += 1;
4954                    let item = TranscriptItem::UserMessage(UserMessage::text(
4955                        RELIABILITY_CONTINUATION_PROMPT.to_string(),
4956                    ));
4957                    self.persist_turn_item(&req.thread_id, &turn_id, &item)
4958                        .await?;
4959                    transcript.push(item);
4960                    continue;
4961                }
4962                if deadline_finalization_requested
4963                    && assistant_text.trim().is_empty()
4964                    && phase_messages.is_empty()
4965                {
4966                    assistant_text = format!(
4967                        "Deadline finalization completed without model text. {}",
4968                        turn_partial_result(&transcript)
4969                    );
4970                }
4971                final_assistant_text = assistant_text;
4972                final_provider_metadata = provider_metadata;
4973                exhausted_tool_rounds = false;
4974                break;
4975            }
4976
4977            if let Some(metadata) = provider_metadata {
4978                let had_provider_compaction =
4979                    crate::compaction::provider_metadata_has_compaction(&metadata);
4980                let item = TranscriptItem::ProviderMetadata(metadata);
4981                self.persist_turn_item(&req.thread_id, &turn_id, &item)
4982                    .await?;
4983                transcript.push(item);
4984                if had_provider_compaction {
4985                    // Server-side compaction replaced the prior window; drop
4986                    // pre-boundary items so subsequent tool rounds use the
4987                    // compact window as the next input.
4988                    transcript = crate::compaction::trim_to_last_compaction_boundary(transcript);
4989                    compacted_this_turn = true;
4990                }
4991            }
4992            eager_tools.drain().await?;
4993            for call in &tool_calls {
4994                if eager_tools.scheduled(&call.id) {
4995                    continue;
4996                }
4997                let tool_item = TranscriptItem::ToolCall(ToolCallRecord {
4998                    id: call.id.clone(),
4999                    name: call.name.clone(),
5000                    arguments: call.arguments.clone(),
5001                });
5002                self.persist_turn_item(&req.thread_id, &turn_id, &tool_item)
5003                    .await?;
5004                transcript.push(tool_item);
5005                self.persist_model_profile_segment(
5006                    &req.thread_id,
5007                    &turn_id,
5008                    model_profile.as_ref(),
5009                    &provider,
5010                    &model,
5011                    "tool_call",
5012                )
5013                .await?;
5014            }
5015            if let Some(deadline) = turn_deadline
5016                && deadline_expired(deadline)
5017            {
5018                self.fail_turn_due_to_deadline(&req.thread_id, &turn_id, deadline, &transcript)
5019                    .await?;
5020                return Ok(TurnRunOutcome::Stopped);
5021            }
5022            let mut results = eager_tools.take_results(&tool_calls);
5023            let remaining_calls = tool_calls
5024                .into_iter()
5025                .filter(|call| !eager_tools.scheduled(&call.id))
5026                .collect();
5027            results.extend(
5028                self.route_tool_calls(
5029                    &req.thread_id,
5030                    &turn_id,
5031                    remaining_calls,
5032                    parallel_tool_calls,
5033                    Some(workspace.as_str()),
5034                    turn_deadline,
5035                )
5036                .await?,
5037            );
5038            let reliability_limit = reliability.record_tool_results(
5039                &cfg.reliability,
5040                &results,
5041                runtime_profile == RuntimeProfile::Interactive,
5042            );
5043            for result in results {
5044                verification_gate.record_tool_result(&result);
5045                transcript.push(TranscriptItem::ToolResult(result));
5046                self.persist_model_profile_segment(
5047                    &req.thread_id,
5048                    &turn_id,
5049                    model_profile.as_ref(),
5050                    &provider,
5051                    &model,
5052                    "tool_result",
5053                )
5054                .await?;
5055            }
5056            if let Some(limit) = reliability_limit {
5057                if limit.decision == ReliabilityLimitDecision::RequestContinuation {
5058                    // Loop persistence: rather than ending the turn, reset the
5059                    // consecutive-failure counter, nudge the model to keep going,
5060                    // and continue the round loop. Bounded by the per-turn tool
5061                    // failure ceiling, `max_model_calls_per_turn`, and
5062                    // `MAX_TOOL_ROUNDS_PER_TURN`.
5063                    self.record_reliability_limit_continuation(
5064                        &req.thread_id,
5065                        &turn_id,
5066                        &provider,
5067                        &model,
5068                        limit,
5069                    )
5070                    .await;
5071                    reliability.reset_consecutive_failures();
5072                    let nudge = TranscriptItem::UserMessage(UserMessage::text(
5073                        RELIABILITY_CONTINUATION_PROMPT.to_string(),
5074                    ));
5075                    self.persist_turn_item(&req.thread_id, &turn_id, &nudge)
5076                        .await?;
5077                    transcript.push(nudge);
5078                } else {
5079                    self.fail_turn_due_to_reliability_limit(
5080                        &req.thread_id,
5081                        &turn_id,
5082                        &provider,
5083                        &model,
5084                        limit,
5085                        &transcript,
5086                    )
5087                    .await?;
5088                    return Ok(TurnRunOutcome::Stopped);
5089                }
5090            }
5091            let compaction_generation_before = self.compaction_generation(&req.thread_id);
5092            transcript = self
5093                .compact_transcript_if_needed(
5094                    &req.thread_id,
5095                    &turn_id,
5096                    &provider,
5097                    &model,
5098                    transcript,
5099                    self.compaction_options_for_turn(&req.thread_id, !compacted_this_turn),
5100                )
5101                .await?;
5102            compacted_this_turn |=
5103                self.compaction_generation(&req.thread_id) != compaction_generation_before;
5104        }
5105
5106        if exhausted_tool_rounds {
5107            let message =
5108                format!("tool call limit reached after {MAX_TOOL_ROUNDS_PER_TURN} rounds");
5109            self.persist_turn_item(
5110                &req.thread_id,
5111                &turn_id,
5112                &TranscriptItem::Error(ErrorRecord {
5113                    message: message.clone(),
5114                }),
5115            )
5116            .await?;
5117            self.emit(RoderEvent::TurnFailed(TurnFailed {
5118                thread_id: req.thread_id.clone(),
5119                turn_id: turn_id.clone(),
5120                error: message.clone(),
5121                error_kind: None,
5122                usage: None,
5123                timestamp: OffsetDateTime::now_utc(),
5124            }))
5125            .await;
5126            self.complete_team_member_turn_with_result(
5127                &req.thread_id,
5128                &turn_id,
5129                TeamMemberStatus::Failed,
5130                None,
5131                Some(message),
5132            )
5133            .await?;
5134            return Ok(TurnRunOutcome::Stopped);
5135        }
5136
5137        if let Some(metadata) = final_provider_metadata {
5138            self.persist_turn_item(
5139                &req.thread_id,
5140                &turn_id,
5141                &TranscriptItem::ProviderMetadata(metadata),
5142            )
5143            .await?;
5144        }
5145
5146        let turn_usage_tokens = turn_usage.total_tokens as i64;
5147        let completed_usage = (!turn_usage.is_empty()).then_some(turn_usage.clone());
5148        self.record_thread_usage_metadata(&req.thread_id, &turn_usage)
5149            .await?;
5150        self.goals
5151            .account_turn_usage(
5152                &req.thread_id,
5153                turn_usage_tokens,
5154                OffsetDateTime::now_utc() - turn_started_at,
5155            )
5156            .await?;
5157        let (cleanup, ownership) = self.await_provider_turn_cleanup(&turn_id).await;
5158        self.record_turn_lifecycle_with_ownership(
5159            req.thread_id.clone(),
5160            turn_id.clone(),
5161            TurnLifecycleState::Completed,
5162            cleanup,
5163            None,
5164            ownership,
5165        )
5166        .await;
5167        self.emit(RoderEvent::TurnCompleted(TurnCompleted {
5168            thread_id: req.thread_id.clone(),
5169            turn_id: turn_id.clone(),
5170            usage: completed_usage,
5171            finish_reason: turn_finish_reason,
5172            timestamp: OffsetDateTime::now_utc(),
5173        }))
5174        .await;
5175        self.complete_team_member_turn_with_result(
5176            &req.thread_id,
5177            &turn_id,
5178            TeamMemberStatus::Completed,
5179            (!final_assistant_text.is_empty()).then_some(final_assistant_text),
5180            None,
5181        )
5182        .await?;
5183        Ok(TurnRunOutcome::Completed)
5184    }
5185
5186    async fn route_tool_calls(
5187        self: &Arc<Self>,
5188        thread_id: &ThreadId,
5189        turn_id: &TurnId,
5190        calls: Vec<ToolCallCompleted>,
5191        parallel: bool,
5192        workspace: Option<&str>,
5193        deadline: Option<OffsetDateTime>,
5194    ) -> anyhow::Result<Vec<ToolResultRecord>> {
5195        // Enforce the agent_swarm exclusivity rule (roadmap 104, Task 2):
5196        // `agent_swarm` must be the only tool call in a model response. A mixed
5197        // or multi-swarm batch is denied wholesale with actionable retry text so
5198        // the model re-issues `agent_swarm` by itself, and every tool_call_id
5199        // still gets a response (keeping the chat-completions transcript valid).
5200        if let Some(violation) = roder_api::subagents::agent_swarm_batch_violation(
5201            calls.iter().map(|call| call.name.as_str()),
5202        ) {
5203            let message = violation.deny_message();
5204            return Ok(calls
5205                .into_iter()
5206                .map(|call| ToolResultRecord {
5207                    id: call.id,
5208                    name: Some(call.name),
5209                    result: message.clone(),
5210                    display_payload: None,
5211                    is_error: true,
5212                })
5213                .collect());
5214        }
5215        // Emit swarm lifecycle events on the bus (roadmap 104, Task 1) so any
5216        // app-server/SDK/TUI client can observe a swarm as a whole; per-child
5217        // progress flows through the existing Subagent* trace events.
5218        let swarm_call = (calls.len() == 1
5219            && calls[0].name == roder_api::subagents::AGENT_SWARM_TOOL_NAME)
5220            .then(|| (calls[0].id.clone(), calls[0].arguments.clone()));
5221        if let Some((tool_id, args)) = &swarm_call {
5222            self.emit(RoderEvent::AgentSwarmStarted(
5223                roder_api::subagents::AgentSwarmStarted {
5224                    thread_id: thread_id.clone(),
5225                    turn_id: turn_id.clone(),
5226                    tool_id: tool_id.clone(),
5227                    child_count: agent_swarm_child_count(args),
5228                    timestamp: OffsetDateTime::now_utc(),
5229                },
5230            ))
5231            .await;
5232        }
5233
5234        let force_sequential = calls
5235            .iter()
5236            .any(|call| crate::agent_control_tools::is_agent_control_tool(&call.name));
5237        let results =
5238            if parallel && !force_sequential {
5239                try_join_all(calls.into_iter().map(|call| {
5240                    self.route_tool_call(thread_id, turn_id, call, workspace, deadline)
5241                }))
5242                .await
5243            } else {
5244                let mut results = Vec::with_capacity(calls.len());
5245                for call in calls {
5246                    results.push(
5247                        self.route_tool_call(thread_id, turn_id, call, workspace, deadline)
5248                            .await?,
5249                    );
5250                }
5251                Ok(results)
5252            }?;
5253
5254        if let Some((tool_id, _)) = &swarm_call
5255            && let Some(result) = results.iter().find(|result| &result.id == tool_id)
5256            && let Some((completed, failed, aborted)) = parse_swarm_counts(&result.result)
5257        {
5258            self.emit(RoderEvent::AgentSwarmCompleted(
5259                roder_api::subagents::AgentSwarmCompleted {
5260                    thread_id: thread_id.clone(),
5261                    turn_id: turn_id.clone(),
5262                    tool_id: tool_id.clone(),
5263                    completed,
5264                    failed,
5265                    aborted,
5266                    timestamp: OffsetDateTime::now_utc(),
5267                },
5268            ))
5269            .await;
5270        }
5271
5272        Ok(results)
5273    }
5274
5275    /// On provider context/prompt-length failures, force-compact (and if needed
5276    /// strip the last bulky item) then retry the current tool round instead of
5277    /// failing the turn. Bounded so a permanently oversized suffix still stops.
5278    async fn try_recover_from_context_limit(
5279        &self,
5280        thread_id: &ThreadId,
5281        turn_id: &TurnId,
5282        provider: &str,
5283        model: &str,
5284        error: &str,
5285        transcript: &mut Vec<TranscriptItem>,
5286        compacted_this_turn: &mut bool,
5287        recovery_attempts: &mut u32,
5288    ) -> anyhow::Result<bool> {
5289        const MAX_CONTEXT_LIMIT_RECOVERY_ATTEMPTS: u32 = 2;
5290        if !crate::compaction::is_context_limit_failure_message(error) {
5291            return Ok(false);
5292        }
5293        if *recovery_attempts >= MAX_CONTEXT_LIMIT_RECOVERY_ATTEMPTS {
5294            return Ok(false);
5295        }
5296        *recovery_attempts = recovery_attempts.saturating_add(1);
5297
5298        let error_item = TranscriptItem::Error(ErrorRecord {
5299            message: error.to_string(),
5300        });
5301        self.persist_turn_item(thread_id, turn_id, &error_item)
5302            .await?;
5303        transcript.push(error_item);
5304
5305        // After the first failed recovery, drop the item ahead of the latest
5306        // user/error so a second compact can succeed when the suffix itself is
5307        // still oversized (e.g. a huge tool result right before the prompt).
5308        if *recovery_attempts > 1 {
5309            *transcript =
5310                crate::compaction::strip_last_message_before_error(std::mem::take(transcript));
5311        }
5312
5313        let force_options = crate::compaction::CompactionOptions {
5314            allow_repeat: true,
5315            force: true,
5316            hysteresis_baseline: None,
5317            preserve_hint: Some(
5318                "Recover from a provider context/prompt-length limit; preserve the active user goal and recent tool outcomes."
5319                    .to_string(),
5320            ),
5321        };
5322        let before_len = transcript.len();
5323        let before_tokens = crate::compaction::estimate_prompt_tokens(transcript);
5324        *transcript = self
5325            .compact_transcript_if_needed(
5326                thread_id,
5327                turn_id,
5328                provider,
5329                model,
5330                std::mem::take(transcript),
5331                force_options,
5332            )
5333            .await?;
5334        let after_tokens = crate::compaction::estimate_prompt_tokens(transcript);
5335        *compacted_this_turn = *compacted_this_turn
5336            || transcript
5337                .iter()
5338                .any(crate::compaction::is_compaction_boundary);
5339
5340        self.emit(RoderEvent::ReliabilityRetryRecorded(
5341            ReliabilityRetryRecorded {
5342                context: ReliabilityContext {
5343                    thread_id: thread_id.clone(),
5344                    turn_id: turn_id.clone(),
5345                    provider: Some(provider.to_string()),
5346                    model: Some(model.to_string()),
5347                    ..ReliabilityContext::default()
5348                },
5349                error_class: ReliabilityErrorClass::ProviderError,
5350                decision: ReliabilityRetryDecision::Retry,
5351                attempt: *recovery_attempts,
5352                max_attempts: MAX_CONTEXT_LIMIT_RECOVERY_ATTEMPTS,
5353                delay_ms: Some(0),
5354                details: ReliabilityDetails::redacted(format!(
5355                    "context_limit_recovery: compact {before_tokens}->{after_tokens} tokens (items {before_len}->{}); {error}",
5356                    transcript.len()
5357                )),
5358                timestamp: OffsetDateTime::now_utc(),
5359            },
5360        ))
5361        .await;
5362
5363        // If force-compact did not shrink anything and we still have budget,
5364        // strip once more so the next round is not identical.
5365        if after_tokens >= before_tokens && *recovery_attempts < MAX_CONTEXT_LIMIT_RECOVERY_ATTEMPTS
5366        {
5367            *transcript =
5368                crate::compaction::strip_last_message_before_error(std::mem::take(transcript));
5369        }
5370        Ok(true)
5371    }
5372
5373    async fn fail_turn_with_error(
5374        &self,
5375        thread_id: &ThreadId,
5376        turn_id: &TurnId,
5377        message: String,
5378    ) -> anyhow::Result<()> {
5379        self.persist_turn_item(
5380            thread_id,
5381            turn_id,
5382            &TranscriptItem::Error(ErrorRecord {
5383                message: message.clone(),
5384            }),
5385        )
5386        .await?;
5387        self.emit(RoderEvent::TurnFailed(TurnFailed {
5388            thread_id: thread_id.clone(),
5389            turn_id: turn_id.clone(),
5390            error: message.clone(),
5391            error_kind: None,
5392            usage: None,
5393            timestamp: OffsetDateTime::now_utc(),
5394        }))
5395        .await;
5396        self.complete_team_member_turn_with_result(
5397            thread_id,
5398            turn_id,
5399            TeamMemberStatus::Failed,
5400            None,
5401            Some(message),
5402        )
5403        .await?;
5404        Ok(())
5405    }
5406
5407    async fn fail_turn_due_to_deadline(
5408        &self,
5409        thread_id: &ThreadId,
5410        turn_id: &TurnId,
5411        deadline: OffsetDateTime,
5412        transcript: &[TranscriptItem],
5413    ) -> anyhow::Result<()> {
5414        let partial_result = turn_partial_result(transcript);
5415        self.emit(RoderEvent::TurnPartialResult(TurnPartialResult {
5416            thread_id: thread_id.clone(),
5417            turn_id: turn_id.clone(),
5418            summary: partial_result.clone(),
5419            timestamp: OffsetDateTime::now_utc(),
5420        }))
5421        .await;
5422        self.emit(RoderEvent::TurnDeadlineExceeded(TurnDeadlineExceeded {
5423            thread_id: thread_id.clone(),
5424            turn_id: turn_id.clone(),
5425            deadline,
5426            partial_result: partial_result.clone(),
5427            timestamp: OffsetDateTime::now_utc(),
5428        }))
5429        .await;
5430        let message = "turn deadline expired".to_string();
5431        self.persist_turn_item(
5432            thread_id,
5433            turn_id,
5434            &TranscriptItem::Error(ErrorRecord {
5435                message: format!("{message}: {partial_result}"),
5436            }),
5437        )
5438        .await?;
5439        self.emit(RoderEvent::TurnFailed(TurnFailed {
5440            thread_id: thread_id.clone(),
5441            turn_id: turn_id.clone(),
5442            error: message.clone(),
5443            error_kind: Some("deadline_timeout".to_string()),
5444            usage: None,
5445            timestamp: OffsetDateTime::now_utc(),
5446        }))
5447        .await;
5448        self.complete_team_member_turn_with_result(
5449            thread_id,
5450            turn_id,
5451            TeamMemberStatus::Failed,
5452            None,
5453            Some(format!("{message}: {partial_result}")),
5454        )
5455        .await?;
5456        Ok(())
5457    }
5458
5459    async fn start_deadline_finalization(
5460        &self,
5461        thread_id: &ThreadId,
5462        turn_id: &TurnId,
5463        transcript: &mut Vec<TranscriptItem>,
5464        remaining_seconds: u64,
5465    ) -> anyhow::Result<()> {
5466        let item = TranscriptItem::UserMessage(crate::deadline_policy::finalization_message(
5467            remaining_seconds,
5468        ));
5469        self.persist_turn_item(thread_id, turn_id, &item).await?;
5470        transcript.push(item);
5471        self.emit(RoderEvent::TurnPartialResult(TurnPartialResult {
5472            thread_id: thread_id.clone(),
5473            turn_id: turn_id.clone(),
5474            summary: turn_partial_result(transcript),
5475            timestamp: OffsetDateTime::now_utc(),
5476        }))
5477        .await;
5478        Ok(())
5479    }
5480
5481    /// Emits the reliability-limit event for a continuation (loop persistence)
5482    /// without failing the turn. The caller resets the failure counter and
5483    /// injects a nudge so the round loop keeps going.
5484    async fn record_reliability_limit_continuation(
5485        &self,
5486        thread_id: &ThreadId,
5487        turn_id: &TurnId,
5488        provider: &str,
5489        model: &str,
5490        limit: ReliabilityLimitHit,
5491    ) {
5492        self.emit(RoderEvent::ReliabilityLimitRecorded(
5493            ReliabilityLimitRecorded {
5494                context: ReliabilityContext {
5495                    thread_id: thread_id.clone(),
5496                    turn_id: turn_id.clone(),
5497                    tool_id: None,
5498                    tool_name: None,
5499                    provider: Some(provider.to_string()),
5500                    model: Some(model.to_string()),
5501                },
5502                error_class: limit.error_class,
5503                limit_kind: limit.limit_kind,
5504                decision: limit.decision,
5505                current: limit.current,
5506                limit: limit.limit,
5507                details: ReliabilityDetails::redacted(&limit.message),
5508                timestamp: OffsetDateTime::now_utc(),
5509            },
5510        ))
5511        .await;
5512    }
5513
5514    async fn fail_turn_due_to_reliability_limit(
5515        &self,
5516        thread_id: &ThreadId,
5517        turn_id: &TurnId,
5518        provider: &str,
5519        model: &str,
5520        limit: ReliabilityLimitHit,
5521        transcript: &[TranscriptItem],
5522    ) -> anyhow::Result<()> {
5523        self.emit(RoderEvent::ReliabilityLimitRecorded(
5524            ReliabilityLimitRecorded {
5525                context: ReliabilityContext {
5526                    thread_id: thread_id.clone(),
5527                    turn_id: turn_id.clone(),
5528                    tool_id: None,
5529                    tool_name: None,
5530                    provider: Some(provider.to_string()),
5531                    model: Some(model.to_string()),
5532                },
5533                error_class: limit.error_class,
5534                limit_kind: limit.limit_kind,
5535                decision: limit.decision,
5536                current: limit.current,
5537                limit: limit.limit,
5538                details: ReliabilityDetails::redacted(&limit.message),
5539                timestamp: OffsetDateTime::now_utc(),
5540            },
5541        ))
5542        .await;
5543        let partial_result = turn_partial_result(transcript);
5544        self.emit(RoderEvent::TurnPartialResult(TurnPartialResult {
5545            thread_id: thread_id.clone(),
5546            turn_id: turn_id.clone(),
5547            summary: partial_result.clone(),
5548            timestamp: OffsetDateTime::now_utc(),
5549        }))
5550        .await;
5551        let message = format!("reliability limit reached: {}", limit.message);
5552        self.persist_turn_item(
5553            thread_id,
5554            turn_id,
5555            &TranscriptItem::Error(ErrorRecord {
5556                message: format!("{message}: {partial_result}"),
5557            }),
5558        )
5559        .await?;
5560        self.emit(RoderEvent::TurnFailed(TurnFailed {
5561            thread_id: thread_id.clone(),
5562            turn_id: turn_id.clone(),
5563            error: message.clone(),
5564            error_kind: Some("reliability_limit".to_string()),
5565            usage: None,
5566            timestamp: OffsetDateTime::now_utc(),
5567        }))
5568        .await;
5569        self.complete_team_member_turn_with_result(
5570            thread_id,
5571            turn_id,
5572            TeamMemberStatus::Failed,
5573            None,
5574            Some(format!("{message}: {partial_result}")),
5575        )
5576        .await?;
5577        Ok(())
5578    }
5579
5580    async fn append_steers(
5581        &self,
5582        req: &StartTurnRequest,
5583        turn_id: &TurnId,
5584        transcript: &mut Vec<TranscriptItem>,
5585        steers: Vec<QueuedTurnSteer>,
5586    ) -> anyhow::Result<()> {
5587        for queued in steers {
5588            let mut steer = queued.message;
5589            steer.text = steer.text.trim().to_string();
5590            if steer.text.is_empty() && steer.images.is_empty() {
5591                continue;
5592            }
5593            let item = TranscriptItem::UserMessage(steer);
5594            self.persist_turn_item(&req.thread_id, turn_id, &item)
5595                .await?;
5596            transcript.push(item);
5597            if let Some(ack) = queued.mailbox_ack {
5598                self.teams
5599                    .mark_mailbox_messages_delivered(&ack.team_id, turn_id, &ack.message_ids)
5600                    .await?;
5601            }
5602        }
5603        Ok(())
5604    }
5605
5606    async fn persist_model_profile_segment(
5607        &self,
5608        thread_id: &ThreadId,
5609        turn_id: &TurnId,
5610        profile: Option<&ModelHarnessProfile>,
5611        provider: &str,
5612        model: &str,
5613        segment: &str,
5614    ) -> anyhow::Result<()> {
5615        let item = TranscriptItem::ProviderMetadata(model_profile_segment_metadata(
5616            profile, provider, model, segment,
5617        ));
5618        self.persist_turn_item(thread_id, turn_id, &item).await
5619    }
5620
5621    fn task_ledger_tool_specs(
5622        &self,
5623        profile: Option<&ModelHarnessProfile>,
5624    ) -> Vec<roder_api::tools::ToolSpec> {
5625        self.tool_registry
5626            .get(TASK_LEDGER_TOOL_NAME)
5627            .map(|tool| {
5628                tool.spec()
5629                    .normalized_for_model_profile(schema_policy_for_model(profile))
5630            })
5631            .into_iter()
5632            .collect()
5633    }
5634
5635    pub(crate) fn engine_for(&self, provider: &str) -> anyhow::Result<Arc<dyn InferenceEngine>> {
5636        self.registry
5637            .inference_engine(provider)
5638            .or_else(|| {
5639                self.registry
5640                    .default_inference_engine()
5641                    .filter(|engine| provider.is_empty() || engine.id() == provider)
5642            })
5643            .ok_or_else(|| anyhow::anyhow!("inference provider {provider:?} is not registered"))
5644    }
5645
5646    pub async fn emit(&self, event: RoderEvent) -> EventEnvelope {
5647        self.dispatch_local_hook_lifecycle(&event).await;
5648        if let Some(record) = Self::lifecycle_record_for_event(&event) {
5649            let _ = self.persist_turn_lifecycle_record(&record).await;
5650        }
5651        let envelope = self.bus.emit(event);
5652        if let (Some(store), Some(thread_id)) = (&self.thread_store, envelope.thread_id.as_ref())
5653            && should_persist_thread_event(thread_id)
5654        {
5655            let _ = store.append_event(thread_id, &envelope).await;
5656        }
5657        self.dispatch_event_sinks(&envelope).await;
5658        envelope
5659    }
5660
5661    async fn dispatch_local_hook_lifecycle(&self, event: &RoderEvent) {
5662        // SessionStart belongs to the session, not the turn: firing it from
5663        // TurnStarted re-ran setup hooks on every user message. ThreadCreated
5664        // and ThreadLoaded carry no turn id, so they borrow a synthetic one the
5665        // way the SessionEnd path does.
5666        let session_turn_id = SESSION_HOOK_TURN_ID.to_string();
5667        let (thread_id, turn_id, name, matcher, input) = match event {
5668            RoderEvent::ThreadCreated(event) => (
5669                &event.thread_id,
5670                &session_turn_id,
5671                "SessionStart",
5672                Some("startup"),
5673                serde_json::json!({"source":"startup"}),
5674            ),
5675            RoderEvent::ThreadLoaded(event) => (
5676                &event.thread_id,
5677                &session_turn_id,
5678                "SessionStart",
5679                Some("resume"),
5680                serde_json::json!({"source":"resume"}),
5681            ),
5682            RoderEvent::TurnCompleted(event) => (
5683                &event.thread_id,
5684                &event.turn_id,
5685                "Stop",
5686                None,
5687                serde_json::json!({"finishReason":event.finish_reason}),
5688            ),
5689            RoderEvent::TurnFailed(event) => (
5690                &event.thread_id,
5691                &event.turn_id,
5692                "Stop",
5693                None,
5694                serde_json::json!({"error":event.error}),
5695            ),
5696            RoderEvent::ContextCompactionStarted(event) => (
5697                &event.thread_id,
5698                &event.turn_id,
5699                "PreCompact",
5700                None,
5701                serde_json::json!({"originalItemCount":event.original_item_count,"originalEstimatedTokens":event.original_estimated_tokens}),
5702            ),
5703            RoderEvent::ContextCompactionRecorded(event) => (
5704                &event.thread_id,
5705                &event.turn_id,
5706                "PostCompact",
5707                None,
5708                serde_json::json!({"compactedItemCount":event.compacted_item_count,"compactedEstimatedTokens":event.compacted_estimated_tokens}),
5709            ),
5710            RoderEvent::SubagentStarted(event) => (
5711                &event.parent_thread_id,
5712                &event.parent_turn_id,
5713                "SubagentStart",
5714                Some(event.agent_type.as_str()),
5715                serde_json::json!({"subagentThreadId":event.thread_id,"agentType":event.agent_type,"description":event.description}),
5716            ),
5717            RoderEvent::SubagentCompleted(event) => (
5718                &event.parent_thread_id,
5719                &event.parent_turn_id,
5720                "SubagentStop",
5721                Some(event.agent_type.as_str()),
5722                serde_json::json!({"subagentThreadId":event.thread_id,"agentType":event.agent_type,"exitReason":event.exit_reason}),
5723            ),
5724            RoderEvent::SubagentFailed(event) => (
5725                &event.parent_thread_id,
5726                &event.parent_turn_id,
5727                "SubagentStop",
5728                Some(event.agent_type.as_str()),
5729                serde_json::json!({"subagentThreadId":event.thread_id,"agentType":event.agent_type,"error":event.error}),
5730            ),
5731            _ => return,
5732        };
5733        // One SessionStart per session, whichever event opened it.
5734        if name == "SessionStart" && !self.claim_session_hook_start(thread_id).await {
5735            return;
5736        }
5737        let workspace = self.config.read().await.workspace.clone();
5738        crate::hooks::run_lifecycle(
5739            self,
5740            thread_id,
5741            turn_id,
5742            workspace.as_deref(),
5743            name,
5744            matcher,
5745            input,
5746        )
5747        .await;
5748    }
5749
5750    /// Claim the single `SessionStart` dispatch for a thread, returning whether
5751    /// this caller won it.
5752    pub(crate) async fn claim_session_hook_start(&self, thread_id: &ThreadId) -> bool {
5753        self.session_hook_started
5754            .write()
5755            .await
5756            .insert(thread_id.clone())
5757    }
5758
5759    /// Records a projected thread item event and persists it through the configured thread store.
5760    ///
5761    /// App-server protocol notification bridges currently call this after translating runtime
5762    /// events into item events. Headless runtime consumers that subscribe to `RoderEvent` directly
5763    /// must make the same call if they need the derived item event stream persisted.
5764    pub async fn record_thread_item_event_kind(
5765        &self,
5766        thread_id: &ThreadId,
5767        turn_id: &TurnId,
5768        timestamp: OffsetDateTime,
5769        kind: ThreadItemEventKind,
5770    ) -> anyhow::Result<ThreadItemEvent> {
5771        let seq = self.next_thread_item_event_seq(thread_id).await?;
5772        let item_event = ThreadItemEvent {
5773            seq,
5774            event_id: uuid::Uuid::new_v4().to_string(),
5775            thread_id: thread_id.clone(),
5776            turn_id: turn_id.clone(),
5777            timestamp,
5778            event: kind,
5779        };
5780        if let Some(store) = &self.thread_store {
5781            store.append_item_event(thread_id, &item_event).await?;
5782        }
5783        self.remember_thread_item_event(&item_event).await?;
5784        Ok(item_event)
5785    }
5786
5787    async fn next_thread_item_event_seq(&self, thread_id: &ThreadId) -> anyhow::Result<u64> {
5788        self.ensure_thread_item_cache(thread_id).await?;
5789        Ok(self
5790            .thread_item_cache
5791            .lock()
5792            .await
5793            .next_item_event_seq(thread_id))
5794    }
5795
5796    pub async fn thread_item_exists(
5797        &self,
5798        thread_id: &ThreadId,
5799        turn_id: &TurnId,
5800        item_id: &str,
5801    ) -> anyhow::Result<bool> {
5802        self.ensure_thread_item_cache(thread_id).await?;
5803        Ok(self
5804            .thread_item_cache
5805            .lock()
5806            .await
5807            .thread_item_exists(thread_id, turn_id, item_id))
5808    }
5809
5810    pub async fn current_reasoning_item_id(
5811        &self,
5812        thread_id: &ThreadId,
5813        turn_id: &TurnId,
5814    ) -> anyhow::Result<Option<String>> {
5815        self.ensure_thread_item_cache(thread_id).await?;
5816        Ok(self
5817            .thread_item_cache
5818            .lock()
5819            .await
5820            .current_reasoning_item_id(thread_id, turn_id))
5821    }
5822
5823    async fn remember_thread_item_event(&self, item_event: &ThreadItemEvent) -> anyhow::Result<()> {
5824        self.ensure_thread_item_cache(&item_event.thread_id).await?;
5825        self.thread_item_cache
5826            .lock()
5827            .await
5828            .remember_item_event(item_event);
5829        Ok(())
5830    }
5831
5832    pub async fn latest_transcript_item_index(
5833        &self,
5834        thread_id: &ThreadId,
5835        turn_id: &TurnId,
5836    ) -> anyhow::Result<Option<usize>> {
5837        self.ensure_thread_item_cache(thread_id).await?;
5838        Ok(self
5839            .thread_item_cache
5840            .lock()
5841            .await
5842            .latest_transcript_item_index(thread_id, turn_id))
5843    }
5844
5845    async fn next_transcript_item_index(
5846        &self,
5847        thread_id: &ThreadId,
5848        turn_id: &TurnId,
5849    ) -> anyhow::Result<usize> {
5850        self.ensure_thread_item_cache(thread_id).await?;
5851        Ok(self
5852            .thread_item_cache
5853            .lock()
5854            .await
5855            .next_transcript_item_index(thread_id, turn_id))
5856    }
5857
5858    async fn remember_transcript_item_index(
5859        &self,
5860        thread_id: &ThreadId,
5861        turn_id: &TurnId,
5862        item_index: usize,
5863    ) -> anyhow::Result<()> {
5864        self.ensure_thread_item_cache(thread_id).await?;
5865        self.thread_item_cache
5866            .lock()
5867            .await
5868            .remember_transcript_item_index(thread_id, turn_id, item_index);
5869        Ok(())
5870    }
5871
5872    async fn ensure_thread_item_cache(&self, thread_id: &ThreadId) -> anyhow::Result<()> {
5873        if self
5874            .thread_item_cache
5875            .lock()
5876            .await
5877            .contains_thread(thread_id)
5878        {
5879            return Ok(());
5880        }
5881
5882        let snapshot = if let Some(store) = &self.thread_store {
5883            store.load_thread(thread_id).await?
5884        } else {
5885            None
5886        };
5887        self.thread_item_cache.lock().await.ensure_thread(
5888            thread_id,
5889            ThreadItemCacheEntry::from_snapshot(snapshot.as_ref()),
5890        );
5891        Ok(())
5892    }
5893
5894    pub(crate) async fn persist_turn_item(
5895        &self,
5896        thread_id: &ThreadId,
5897        turn_id: &TurnId,
5898        item: &TranscriptItem,
5899    ) -> anyhow::Result<()> {
5900        let item_index = self.next_transcript_item_index(thread_id, turn_id).await?;
5901        let timestamp = OffsetDateTime::now_utc();
5902        self.emit(RoderEvent::TranscriptItemAppended(TranscriptItemAppended {
5903            thread_id: thread_id.clone(),
5904            turn_id: turn_id.clone(),
5905            item_type: match item {
5906                TranscriptItem::UserMessage(_) => "user_message",
5907                TranscriptItem::AssistantMessage(_) => "assistant_message",
5908                TranscriptItem::ReasoningSummary(_) => "reasoning_summary",
5909                TranscriptItem::ToolCall(_) => "tool_call",
5910                TranscriptItem::ToolResult(_) => "tool_result",
5911                TranscriptItem::FileChange(_) => "file_change",
5912                TranscriptItem::ContextCompaction(_) => "context_compaction",
5913                TranscriptItem::Error(_) => "error",
5914                TranscriptItem::ProviderMetadata(_) => "provider_metadata",
5915            }
5916            .to_string(),
5917            item_index: Some(item_index),
5918            item: Some(item.clone()),
5919            timestamp,
5920        }))
5921        .await;
5922        self.remember_transcript_item_index(thread_id, turn_id, item_index)
5923            .await?;
5924        Ok(())
5925    }
5926}
5927
5928fn transcript_has_images(transcript: &[TranscriptItem]) -> bool {
5929    transcript.iter().any(|item| {
5930        matches!(
5931            item,
5932            TranscriptItem::UserMessage(message) if !message.images.is_empty()
5933        )
5934    })
5935}
5936
5937fn transcript_has_task_ledger(transcript: &[TranscriptItem]) -> bool {
5938    transcript.iter().any(|item| {
5939        matches!(
5940            item,
5941            TranscriptItem::ToolResult(result)
5942                if result.name.as_deref() == Some(TASK_LEDGER_TOOL_NAME) && !result.is_error
5943        )
5944    })
5945}
5946
5947fn task_ledger_completion_prompt(transcript: &[TranscriptItem]) -> Option<String> {
5948    let latest = transcript.iter().rev().find_map(|item| match item {
5949        TranscriptItem::ToolResult(result)
5950            if result.name.as_deref() == Some(TASK_LEDGER_TOOL_NAME) && !result.is_error =>
5951        {
5952            Some(result.result.as_str())
5953        }
5954        _ => None,
5955    })?;
5956    if !task_ledger_has_open_items(latest) {
5957        return None;
5958    }
5959
5960    let mut ledger = latest.chars().take(1500).collect::<String>();
5961    if latest.chars().nth(1500).is_some() {
5962        ledger.push_str("...");
5963    }
5964    Some(format!(
5965        "Task Ledger Completion Required: the latest task ledger still has pending or in-progress items. Do not provide a final answer yet. Use tools to complete the remaining scoreable work, create or update any required output files, then call `{TASK_LEDGER_TOOL_NAME}` with every task completed and evidence before finalizing.\n\nLatest ledger:\n{ledger}"
5966    ))
5967}
5968
5969fn task_ledger_deadline_completion_prompt(
5970    remaining_seconds: u64,
5971    reserve_seconds: u64,
5972    completion_prompt: &str,
5973) -> String {
5974    format!(
5975        "Eval deadline scoreable completion: {remaining_seconds} seconds remain in the {reserve_seconds}-second finalization reserve. Do not browse, search, or start slow work. Use the available tools now to create or update the required scoreable output files, run only a quick local check if needed, then update the task ledger to completed before finalizing.\n\n{completion_prompt}"
5976    )
5977}
5978
5979fn task_ledger_scoreable_checkpoint_prompt(
5980    remaining_seconds: u64,
5981    completion_prompt: &str,
5982) -> String {
5983    format!(
5984        "Scoreable Output Checkpoint: {remaining_seconds} seconds remain before the eval deadline. Before any further research, browsing, or long commands, use tools now to ensure the required output file(s) exist with the best evidence-backed answer, even if provisional. If a scoreable file already exists, read it and preserve that candidate unless you have stronger task-specific evidence for a replacement. Do not overwrite a plausible dated, historical, or local-evidence candidate with a current live-page, partial-coverage, or weaker guess merely to refresh the checkpoint. You may continue refining afterward, but do not apologize or finalize until the scoreable file exists and the task ledger is updated.\n\n{completion_prompt}"
5985    )
5986}
5987
5988fn task_ledger_has_open_items(ledger: &str) -> bool {
5989    ledger.lines().any(|line| {
5990        let line = line.trim_start();
5991        line.starts_with("- pending:") || line.starts_with("- in_progress:")
5992    })
5993}
5994
5995fn turn_deadline_for_config(cfg: &RuntimeConfig) -> Option<OffsetDateTime> {
5996    cfg.turn_deadline_seconds
5997        .filter(|seconds| *seconds > 0)
5998        .map(|seconds| OffsetDateTime::now_utc() + Duration::seconds(seconds as i64))
5999}
6000
6001fn deadline_expired(deadline: OffsetDateTime) -> bool {
6002    OffsetDateTime::now_utc() >= deadline
6003}
6004
6005pub(crate) fn deadline_remaining_seconds(deadline: Option<OffsetDateTime>) -> Option<u64> {
6006    let deadline = deadline?;
6007    if deadline <= OffsetDateTime::now_utc() {
6008        return Some(0);
6009    }
6010    Some(
6011        (deadline - OffsetDateTime::now_utc())
6012            .unsigned_abs()
6013            .as_secs()
6014            .max(1),
6015    )
6016}
6017
6018fn deadline_instant(deadline: OffsetDateTime) -> tokio::time::Instant {
6019    let now = OffsetDateTime::now_utc();
6020    if deadline <= now {
6021        return tokio::time::Instant::now();
6022    }
6023    tokio::time::Instant::now() + (deadline - now).unsigned_abs()
6024}
6025
6026fn inference_timeout_deadline(
6027    deadline: Option<OffsetDateTime>,
6028    runtime_profile: RuntimeProfile,
6029    task_ledger_required: bool,
6030    reserve_seconds: u64,
6031    finalization_requested: bool,
6032    task_ledger_scoreable_checkpoints: u8,
6033    transcript: &[TranscriptItem],
6034) -> Option<(OffsetDateTime, InferenceTimeoutAction)> {
6035    let deadline = deadline?;
6036    if runtime_profile == RuntimeProfile::Eval
6037        && task_ledger_required
6038        && !finalization_requested
6039        && task_ledger_scoreable_checkpoints < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
6040        && TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS > reserve_seconds
6041        && task_ledger_completion_prompt(transcript).is_some()
6042    {
6043        let checkpoint_deadline =
6044            deadline - Duration::seconds(TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS as i64);
6045        if checkpoint_deadline > OffsetDateTime::now_utc() {
6046            return Some((
6047                checkpoint_deadline,
6048                InferenceTimeoutAction::ScoreableCheckpoint,
6049            ));
6050        }
6051    }
6052    if runtime_profile == RuntimeProfile::Eval && !finalization_requested {
6053        return Some((
6054            deadline - Duration::seconds(reserve_seconds as i64),
6055            InferenceTimeoutAction::Finalization,
6056        ));
6057    }
6058    Some((deadline, InferenceTimeoutAction::Finalization))
6059}
6060
6061fn turn_partial_result(transcript: &[TranscriptItem]) -> String {
6062    let tool_results = transcript
6063        .iter()
6064        .filter(|item| matches!(item, TranscriptItem::ToolResult(_)))
6065        .count();
6066    let assistant_messages = transcript
6067        .iter()
6068        .filter(|item| matches!(item, TranscriptItem::AssistantMessage(_)))
6069        .count();
6070    format!(
6071        "partial turn state: {} transcript items, {assistant_messages} assistant messages, {tool_results} tool results",
6072        transcript.len()
6073    )
6074}
6075
6076fn reasoning_for_model(cfg: &RuntimeConfig, model: &str) -> ReasoningConfig {
6077    let level = effective_reasoning_for_model(cfg, model);
6078    match level.as_str() {
6079        "" | REASONING_NONE => ReasoningConfig::default(),
6080        level => ReasoningConfig {
6081            enabled: true,
6082            level: Some(level.to_string()),
6083        },
6084    }
6085}
6086
6087fn server_side_compaction_threshold(cfg: &RuntimeConfig, model: &str) -> Option<u32> {
6088    let entry = lookup_model(model)?;
6089    if !entry.supports_compaction {
6090        return None;
6091    }
6092    cfg.auto_compact_token_limit
6093        .or_else(|| {
6094            model_profile_for_model(cfg, model).and_then(|profile| profile.auto_compact_token_limit)
6095        })
6096        .or(Some(entry.auto_compact_token_limit))
6097        .filter(|threshold| *threshold > 0)
6098}
6099
6100pub(crate) fn tool_search_for_provider_model(
6101    cfg: &RuntimeConfig,
6102    provider: &str,
6103    model: &str,
6104) -> ToolSearchConfig {
6105    let mut resolved = cfg.tool_search.clone();
6106    if let Some(provider_config) = cfg.provider_tool_search.get(provider) {
6107        provider_config.apply_to(&mut resolved);
6108    }
6109    if let Some(model_config) = cfg.model_tool_search.get(model) {
6110        model_config.apply_to(&mut resolved);
6111    }
6112    resolved
6113}
6114
6115fn parallel_tool_calls_for_model(cfg: &RuntimeConfig, model: &str) -> bool {
6116    cfg.model_parallel_tool_calls
6117        .get(model)
6118        .copied()
6119        .or_else(|| {
6120            model_profile_for_model(cfg, model).and_then(|profile| profile.parallel_tool_calls)
6121        })
6122        .unwrap_or(true)
6123}
6124
6125/// Count the children an `agent_swarm` call will launch from its arguments
6126/// (item-based spawns plus resumes). Lenient: returns 0 on malformed input.
6127fn agent_swarm_child_count(arguments: &str) -> usize {
6128    serde_json::from_str::<roder_api::subagents::AgentSwarmRequest>(arguments)
6129        .map(|request| request.items.len() + request.resume_agent_ids.len())
6130        .unwrap_or(0)
6131}
6132
6133/// Parse `completed`/`failed`/`aborted` counts from an `<agent_swarm_result>`
6134/// summary line, e.g. `<summary>completed: 2, failed: 1</summary>`. Omitted
6135/// buckets default to 0. Returns `None` when the text is not a swarm result.
6136fn parse_swarm_counts(text: &str) -> Option<(usize, usize, usize)> {
6137    if !text.contains("<agent_swarm_result>") {
6138        return None;
6139    }
6140    let bucket = |label: &str| -> usize {
6141        let needle = format!("{label}: ");
6142        text.find(&needle)
6143            .map(|start| start + needle.len())
6144            .map(|start| {
6145                text[start..]
6146                    .chars()
6147                    .take_while(|c| c.is_ascii_digit())
6148                    .collect::<String>()
6149            })
6150            .and_then(|digits| digits.parse().ok())
6151            .unwrap_or(0)
6152    };
6153    Some((bucket("completed"), bucket("failed"), bucket("aborted")))
6154}
6155
6156fn effective_reasoning_for_model(cfg: &RuntimeConfig, model: &str) -> String {
6157    let base_reasoning = default_effective_reasoning_for_model(cfg, model);
6158    if cfg.dynamic_workflows.effort_profile == DynamicWorkflowEffortProfile::Ultracode {
6159        return ultracode_reasoning_level_for_model(
6160            model,
6161            &cfg.speed_policy.ultracode_reasoning,
6162            &base_reasoning,
6163        );
6164    }
6165    base_reasoning
6166}
6167
6168fn default_effective_reasoning_for_model(cfg: &RuntimeConfig, model: &str) -> String {
6169    let Some(entry) = lookup_model(model) else {
6170        return cfg
6171            .reasoning
6172            .clone()
6173            .unwrap_or_else(|| REASONING_NONE.to_string());
6174    };
6175    if entry.supported_reasoning.is_empty() {
6176        return REASONING_NONE.to_string();
6177    }
6178    cfg.reasoning
6179        .as_deref()
6180        .filter(|reasoning| {
6181            entry
6182                .supported_reasoning
6183                .iter()
6184                .any(|option| option.effort == *reasoning)
6185        })
6186        .map(str::to_string)
6187        .or_else(|| {
6188            model_profile_for_model(cfg, model)
6189                .and_then(|profile| profile.reasoning.orientation)
6190                .filter(|reasoning| {
6191                    entry
6192                        .supported_reasoning
6193                        .iter()
6194                        .any(|option| option.effort == reasoning)
6195                })
6196        })
6197        .unwrap_or_else(|| entry.default_reasoning.to_string())
6198}
6199
6200fn validate_reasoning_effort(model: &str, effort: &str) -> anyhow::Result<()> {
6201    if effort == REASONING_NONE && !model_supports_reasoning(model, effort) {
6202        return Ok(());
6203    }
6204    let Some(entry) = lookup_model(model) else {
6205        return Ok(());
6206    };
6207    if entry
6208        .supported_reasoning
6209        .iter()
6210        .any(|option| option.effort == effort)
6211    {
6212        Ok(())
6213    } else {
6214        anyhow::bail!("model {model} does not support reasoning effort {effort}")
6215    }
6216}
6217
6218fn validate_runtime_config_reasoning(cfg: &RuntimeConfig) -> anyhow::Result<()> {
6219    let Some(reasoning) = cfg.reasoning.as_deref() else {
6220        return Ok(());
6221    };
6222    let Some(entry) = lookup_model(&cfg.default_model) else {
6223        return Ok(());
6224    };
6225    if entry.provider != PROVIDER_GEMINI {
6226        return Ok(());
6227    }
6228    validate_reasoning_effort(&cfg.default_model, reasoning)
6229}
6230
6231fn validate_runtime_inference_router_config(
6232    registry: &ExtensionRegistry,
6233    cfg: &RuntimeConfig,
6234) -> anyhow::Result<()> {
6235    if !cfg.inference_router.enabled {
6236        return Ok(());
6237    }
6238    let Some(router_id) = cfg.inference_router.router_id.as_deref() else {
6239        anyhow::bail!("inference_router.enabled requires inference_router.router");
6240    };
6241    if registry.inference_router(router_id).is_some() {
6242        return Ok(());
6243    }
6244    let available = registry
6245        .inference_routers
6246        .iter()
6247        .map(|router| router.id())
6248        .collect::<Vec<_>>()
6249        .join(", ");
6250    if available.is_empty() {
6251        anyhow::bail!("inference router {router_id:?} is not registered");
6252    }
6253    anyhow::bail!(
6254        "inference router {router_id:?} is not registered; available routers: {available}"
6255    );
6256}
6257
6258fn model_supports_reasoning(model: &str, effort: &str) -> bool {
6259    lookup_model(model)
6260        .map(|entry| {
6261            entry
6262                .supported_reasoning
6263                .iter()
6264                .any(|option| option.effort == effort)
6265        })
6266        .unwrap_or(false)
6267}
6268
6269fn is_final_answer_phase(phase: Option<&str>) -> bool {
6270    phase.is_none_or(|phase| phase.is_empty() || phase == FINAL_ANSWER_PHASE)
6271}
6272
6273fn edit_tool_for_model<'a>(cfg: &'a RuntimeConfig, model: &'a str) -> Option<&'a str> {
6274    cfg.model_edit_tools
6275        .get(model)
6276        .map(String::as_str)
6277        .or_else(|| {
6278            cfg.model_profiles
6279                .get(model)
6280                .and_then(|profile| profile.edit_tool.as_deref())
6281        })
6282        .or_else(|| lookup_model(model).and_then(|entry| entry.edit_tool))
6283        .or(Some(EDIT_TOOL_EDIT))
6284}
6285
6286fn model_profile_for_model(cfg: &RuntimeConfig, model: &str) -> Option<ModelHarnessProfile> {
6287    cfg.model_profiles
6288        .get(model)
6289        .cloned()
6290        .or_else(|| built_in_model_profile(model))
6291}
6292
6293pub(crate) fn allowlist_permits(allowlist: &[String], tool_name: &str) -> bool {
6294    allowlist.is_empty() || allowlist.iter().any(|allowed| allowed == tool_name)
6295}
6296
6297/// Provider-aware profile resolution for the active turn.
6298///
6299/// Many model ids are shared across providers (for example Cursor proxies
6300/// `claude-opus-4-8`). Resolving the harness profile by id alone picks the
6301/// first catalog entry, which assigns cross-provider ids the wrong family and
6302/// instruction overlay (e.g. a `cursor/claude-opus-4-8` turn would otherwise
6303/// inherit the Anthropic overlay). Prefer the explicit `(provider, model)`
6304/// catalog entry, keeping user-configured profile overrides as the top
6305/// precedence.
6306fn model_profile_for_provider_model(
6307    cfg: &RuntimeConfig,
6308    provider: &str,
6309    model: &str,
6310) -> Option<ModelHarnessProfile> {
6311    cfg.model_profiles
6312        .get(model)
6313        .cloned()
6314        .or_else(|| built_in_model_profile_for_provider(provider, model))
6315}
6316
6317fn schema_policy_for_model(profile: Option<&ModelHarnessProfile>) -> ModelSchemaPolicy {
6318    profile
6319        .map(|profile| profile.schema_policy)
6320        .unwrap_or_default()
6321}
6322
6323fn model_profile_segment_metadata(
6324    profile: Option<&ModelHarnessProfile>,
6325    provider: &str,
6326    model: &str,
6327    segment: &str,
6328) -> serde_json::Value {
6329    serde_json::json!({
6330        "kind": MODEL_PROFILE_TRACE_KIND,
6331        "segment": segment,
6332        "provider": provider,
6333        "model": model,
6334        "profileModel": profile.map(|profile| profile.model.as_str()).unwrap_or(model),
6335        "providerFamily": profile.map(|profile| profile.provider_family),
6336        "editTool": profile.and_then(|profile| profile.edit_tool.as_deref()),
6337        "schemaPolicy": profile.map(|profile| profile.schema_policy),
6338        "instructionOverlay": profile.map(|profile| profile.instruction_overlay),
6339        "parallelToolCalls": profile.and_then(|profile| profile.parallel_tool_calls),
6340        "autoCompactTokenLimit": profile.and_then(|profile| profile.auto_compact_token_limit),
6341    })
6342}
6343
6344fn model_switch_summary(
6345    transcript: &[TranscriptItem],
6346    profile: Option<&ModelHarnessProfile>,
6347    provider: &str,
6348    model: &str,
6349    tools: &[roder_api::tools::ToolSpec],
6350) -> Option<String> {
6351    let previous = latest_model_profile_segment(transcript)?;
6352    let previous_model = previous
6353        .get("model")
6354        .and_then(serde_json::Value::as_str)
6355        .unwrap_or_default();
6356    let previous_provider = previous
6357        .get("provider")
6358        .and_then(serde_json::Value::as_str)
6359        .unwrap_or_default();
6360    if previous_model == model && previous_provider == provider {
6361        return None;
6362    }
6363
6364    let previous_profile = previous
6365        .get("profileModel")
6366        .and_then(serde_json::Value::as_str)
6367        .unwrap_or(previous_model);
6368    let current_profile = profile
6369        .map(|profile| profile.model.as_str())
6370        .unwrap_or(model);
6371    let previous_edit_tool = previous
6372        .get("editTool")
6373        .and_then(serde_json::Value::as_str)
6374        .unwrap_or("none");
6375    let current_edit_tool = profile
6376        .and_then(|profile| profile.edit_tool.as_deref())
6377        .unwrap_or("none");
6378    let tool_names = tools
6379        .iter()
6380        .map(|tool| tool.name.as_str())
6381        .take(12)
6382        .collect::<Vec<_>>()
6383        .join(", ");
6384    Some(format!(
6385        "{MODEL_SWITCH_SUMMARY_PREFIX} previous profile {previous_provider}/{previous_profile} used edit tool {previous_edit_tool}. Current profile {provider}/{current_profile} uses edit tool {current_edit_tool}. Available tools now: {}.",
6386        if tool_names.is_empty() {
6387            "none"
6388        } else {
6389            &tool_names
6390        }
6391    ))
6392}
6393
6394fn latest_model_profile_segment(transcript: &[TranscriptItem]) -> Option<&serde_json::Value> {
6395    transcript.iter().rev().find_map(|item| {
6396        let TranscriptItem::ProviderMetadata(value) = item else {
6397            return None;
6398        };
6399        (value.get("kind").and_then(serde_json::Value::as_str) == Some(MODEL_PROFILE_TRACE_KIND))
6400            .then_some(value)
6401    })
6402}
6403
6404pub fn validate_edit_tool(value: &str) -> anyhow::Result<()> {
6405    match value.trim() {
6406        EDIT_TOOL_PATCH | EDIT_TOOL_EDIT => Ok(()),
6407        _ => anyhow::bail!(
6408            "unsupported edit_tool {value:?}; allowed values: {EDIT_TOOL_PATCH}, {EDIT_TOOL_EDIT}"
6409        ),
6410    }
6411}
6412
6413fn should_persist_thread_event(thread_id: &str) -> bool {
6414    !is_synthetic_event_thread_id(thread_id)
6415}
6416
6417#[cfg(test)]
6418#[path = "runtime/item_identity_tests.rs"]
6419mod item_identity_tests;
6420
6421#[cfg(test)]
6422#[path = "runtime/codex_v2_tests.rs"]
6423mod codex_v2_tests;
6424
6425#[cfg(test)]
6426#[path = "runtime/codex_v2_lifecycle_tests.rs"]
6427mod codex_v2_lifecycle_tests;
6428
6429#[cfg(test)]
6430mod tests {
6431    use super::*;
6432    use futures::stream;
6433    use roder_api::catalog::{
6434        PROVIDER_MOCK, REASONING_HIGH, REASONING_LOW, REASONING_MEDIUM, REASONING_MINIMAL,
6435        REASONING_NONE, REASONING_XHIGH,
6436    };
6437    use roder_api::extension::ExtensionRegistryBuilder;
6438    use roder_api::inference::{
6439        CompletionMetadata, InferenceCapabilities, InferenceEngine, InferenceEventStream,
6440        InferenceProviderContext, InferenceTurnContext, MessageDelta, ModelDescriptor,
6441        ModelInstructionOverlay, ModelProfileReasoning, ModelSchemaPolicy, ProviderFamily,
6442        ReasoningEffortDescriptor,
6443    };
6444    use roder_api::inference_routing::{
6445        InferenceRouter, InferenceRoutingContext, InferenceRoutingDecision, InferenceRoutingOutcome,
6446    };
6447    use roder_api::thread::ThreadStoreFactory;
6448    use roder_api::tools::{ToolContributor, ToolExecutor, ToolSpec};
6449    use roder_ext_jsonl_thread_store::store::JsonlThreadStoreFactory;
6450    use std::sync::Mutex as StdMutex;
6451
6452    fn test_workspace() -> String {
6453        std::env::current_dir().unwrap().display().to_string()
6454    }
6455
6456    #[test]
6457    fn interactive_explicit_turn_deadline_has_bounded_remaining_time() {
6458        let config = RuntimeConfig {
6459            runtime_profile: RuntimeProfile::Interactive,
6460            turn_deadline_seconds: Some(60),
6461            ..RuntimeConfig::default()
6462        };
6463
6464        let deadline = turn_deadline_for_config(&config).expect("configured turn deadline");
6465        let remaining = deadline_remaining_seconds(Some(deadline));
6466
6467        assert!(
6468            matches!(remaining, Some(1..=60)),
6469            "interactive deadline must be positive and no longer than configured: {remaining:?}"
6470        );
6471    }
6472
6473    #[test]
6474    fn interactive_without_turn_deadline_stays_unbounded() {
6475        let config = RuntimeConfig {
6476            runtime_profile: RuntimeProfile::Interactive,
6477            turn_deadline_seconds: None,
6478            ..RuntimeConfig::default()
6479        };
6480
6481        assert_eq!(turn_deadline_for_config(&config), None);
6482    }
6483
6484    struct MetadataMissingStore;
6485
6486    #[async_trait::async_trait]
6487    impl ThreadStore for MetadataMissingStore {
6488        fn id(&self) -> roder_api::thread::ThreadStoreId {
6489            "metadata-missing-store".to_string()
6490        }
6491
6492        async fn create_thread(&self, metadata: ThreadMetadata) -> anyhow::Result<ThreadMetadata> {
6493            Ok(metadata)
6494        }
6495
6496        async fn list_threads(&self) -> anyhow::Result<Vec<ThreadMetadata>> {
6497            Ok(Vec::new())
6498        }
6499
6500        async fn load_thread(
6501            &self,
6502            _thread_id: &ThreadId,
6503        ) -> anyhow::Result<Option<ThreadSnapshot>> {
6504            Ok(Some(ThreadSnapshot {
6505                metadata: None,
6506                ..ThreadSnapshot::default()
6507            }))
6508        }
6509
6510        async fn append_event(
6511            &self,
6512            _thread_id: &ThreadId,
6513            _envelope: &EventEnvelope,
6514        ) -> anyhow::Result<()> {
6515            Ok(())
6516        }
6517    }
6518
6519    struct MetadataMissingStoreFactory;
6520
6521    impl ThreadStoreFactory for MetadataMissingStoreFactory {
6522        fn id(&self) -> roder_api::thread::ThreadStoreId {
6523            "metadata-missing-store".to_string()
6524        }
6525
6526        fn create(&self) -> Arc<dyn ThreadStore> {
6527            Arc::new(MetadataMissingStore)
6528        }
6529    }
6530
6531    #[test]
6532    fn synthetic_app_server_events_are_not_thread_events() {
6533        for thread_id in ["app-server", "runtime", "thread-workflow"] {
6534            assert!(!should_persist_thread_event(thread_id));
6535        }
6536        assert!(should_persist_thread_event("thread-discovery"));
6537        assert!(should_persist_thread_event("thread-plan"));
6538        assert!(should_persist_thread_event("thread-process"));
6539        assert!(should_persist_thread_event("thread-1"));
6540    }
6541
6542    #[test]
6543    fn server_side_compaction_uses_catalog_ninety_percent_default() {
6544        assert_eq!(
6545            server_side_compaction_threshold(&RuntimeConfig::default(), "gpt-5.5"),
6546            Some(945_000)
6547        );
6548        assert_eq!(
6549            server_side_compaction_threshold(&RuntimeConfig::default(), "gpt-5.3-codex-spark"),
6550            Some(115_200)
6551        );
6552    }
6553
6554    #[test]
6555    fn server_side_compaction_respects_explicit_config_override() {
6556        let cfg = RuntimeConfig {
6557            auto_compact_token_limit: Some(123_456),
6558            ..RuntimeConfig::default()
6559        };
6560
6561        assert_eq!(
6562            server_side_compaction_threshold(&cfg, "gpt-5.5"),
6563            Some(123_456)
6564        );
6565    }
6566
6567    #[tokio::test]
6568    async fn pre_request_compaction_runs_when_server_side_model_is_at_context_window() {
6569        let captured = Arc::new(StdMutex::new(None));
6570        let mut builder = ExtensionRegistryBuilder::new();
6571        builder.inference_engine(Arc::new(CapturingEngine {
6572            request: captured.clone(),
6573        }));
6574        let thread_root = std::env::temp_dir().join(format!(
6575            "roder-pre-request-compaction-{}",
6576            uuid::Uuid::new_v4()
6577        ));
6578        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
6579            base_path: thread_root.clone(),
6580        }));
6581        let runtime = Arc::new(
6582            Runtime::new(
6583                builder.build().unwrap(),
6584                RuntimeConfig {
6585                    default_provider: PROVIDER_MOCK.to_string(),
6586                    default_model: "gpt-5.5".to_string(),
6587                    file_backed_dynamic_context: true,
6588                    ..RuntimeConfig::default()
6589                },
6590            )
6591            .unwrap(),
6592        );
6593        let thread_id = runtime
6594            .create_thread(Some("Pre-request compaction".to_string()))
6595            .await
6596            .unwrap()
6597            .thread_id;
6598        let old_turn = "old-turn".to_string();
6599        runtime
6600            .persist_turn_item(
6601                &thread_id,
6602                &old_turn,
6603                &TranscriptItem::UserMessage(UserMessage::text("old context ".repeat(4_300_000))),
6604            )
6605            .await
6606            .unwrap();
6607
6608        let mut events = runtime.subscribe_events();
6609        runtime
6610            .start_turn(StartTurnRequest {
6611                thread_id: thread_id.clone(),
6612                message: "continue".to_string(),
6613                images: Vec::new(),
6614                provider_override: None,
6615                model_override: None,
6616                reasoning_override: None,
6617                workspace: test_workspace(),
6618                instructions: InstructionBundle::default(),
6619                developer_context: None,
6620                task_ledger_required: false,
6621                service_tier_override: None,
6622            })
6623            .await
6624            .unwrap();
6625        loop {
6626            let envelope = tokio::time::timeout(std::time::Duration::from_secs(5), events.recv())
6627                .await
6628                .unwrap()
6629                .unwrap();
6630            if envelope.thread_id.as_deref() == Some(&thread_id)
6631                && matches!(envelope.event, RoderEvent::TurnCompleted(_))
6632            {
6633                break;
6634            }
6635        }
6636
6637        let request = captured.lock().unwrap().clone().unwrap();
6638        assert!(
6639            matches!(
6640                request.transcript.first(),
6641                Some(TranscriptItem::ContextCompaction(_))
6642            ),
6643            "provider request should start with a local emergency compaction item"
6644        );
6645        assert!(
6646            request.transcript.len() < 4,
6647            "provider request should not replay the full oversized prior transcript: {:?}",
6648            request.transcript
6649        );
6650
6651        let _ = std::fs::remove_dir_all(thread_root);
6652    }
6653
6654    #[tokio::test]
6655    async fn continue_after_context_window_failure_compacts_before_provider_request() {
6656        let captured = Arc::new(StdMutex::new(None));
6657        let mut builder = ExtensionRegistryBuilder::new();
6658        builder.inference_engine(Arc::new(CapturingEngine {
6659            request: captured.clone(),
6660        }));
6661        let thread_root = std::env::temp_dir().join(format!(
6662            "roder-context-failure-continue-{}",
6663            uuid::Uuid::new_v4()
6664        ));
6665        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
6666            base_path: thread_root.clone(),
6667        }));
6668        let runtime = Arc::new(
6669            Runtime::new(
6670                builder.build().unwrap(),
6671                RuntimeConfig {
6672                    default_provider: PROVIDER_MOCK.to_string(),
6673                    default_model: "gpt-5.5".to_string(),
6674                    file_backed_dynamic_context: true,
6675                    ..RuntimeConfig::default()
6676                },
6677            )
6678            .unwrap(),
6679        );
6680        let thread_id = runtime
6681            .create_thread(Some("Context failure continue".to_string()))
6682            .await
6683            .unwrap()
6684            .thread_id;
6685        let failed_turn = "failed-turn".to_string();
6686        runtime
6687            .persist_turn_item(
6688                &thread_id,
6689                &failed_turn,
6690                &TranscriptItem::UserMessage(UserMessage::text("old work ".repeat(10_000))),
6691            )
6692            .await
6693            .unwrap();
6694        runtime
6695            .persist_turn_item(
6696                &thread_id,
6697                &failed_turn,
6698                &TranscriptItem::Error(ErrorRecord {
6699                    message: "Your input exceeds the context window of this model. Please adjust your input and try again."
6700                        .to_string(),
6701                }),
6702            )
6703            .await
6704            .unwrap();
6705
6706        let mut events = runtime.subscribe_events();
6707        runtime
6708            .start_turn(StartTurnRequest {
6709                thread_id: thread_id.clone(),
6710                message: "continue".to_string(),
6711                images: Vec::new(),
6712                provider_override: None,
6713                model_override: None,
6714                reasoning_override: None,
6715                workspace: test_workspace(),
6716                instructions: InstructionBundle::default(),
6717                developer_context: None,
6718                task_ledger_required: false,
6719                service_tier_override: None,
6720            })
6721            .await
6722            .unwrap();
6723        loop {
6724            let envelope = tokio::time::timeout(std::time::Duration::from_secs(5), events.recv())
6725                .await
6726                .unwrap()
6727                .unwrap();
6728            if envelope.thread_id.as_deref() == Some(&thread_id)
6729                && matches!(envelope.event, RoderEvent::TurnCompleted(_))
6730            {
6731                break;
6732            }
6733        }
6734
6735        let request = captured.lock().unwrap().clone().unwrap();
6736        assert!(
6737            matches!(
6738                request.transcript.first(),
6739                Some(TranscriptItem::ContextCompaction(_))
6740            ),
6741            "provider request after context-window failure should start with local compaction"
6742        );
6743        assert!(
6744            request
6745                .transcript
6746                .iter()
6747                .any(|item| matches!(item, TranscriptItem::UserMessage(message) if message.text == "continue")),
6748            "current continue prompt must be preserved: {:?}",
6749            request.transcript
6750        );
6751        assert!(
6752            !request.transcript.iter().any(
6753                |item| matches!(item, TranscriptItem::Error(error) if error.message.contains("context window"))
6754            ),
6755            "raw prior context-window error should be summarized, not replayed: {:?}",
6756            request.transcript
6757        );
6758
6759        let _ = std::fs::remove_dir_all(thread_root);
6760    }
6761
6762    #[tokio::test]
6763    async fn workspace_for_thread_falls_back_when_metadata_is_missing() {
6764        let workspace = test_workspace();
6765        let mut builder = ExtensionRegistryBuilder::new();
6766        builder.inference_engine(Arc::new(FakeInferenceEngine));
6767        builder.thread_store_factory(Arc::new(MetadataMissingStoreFactory));
6768        let runtime = Runtime::new(
6769            builder.build().unwrap(),
6770            RuntimeConfig {
6771                workspace: Some(workspace.clone()),
6772                ..RuntimeConfig::default()
6773            },
6774        )
6775        .unwrap();
6776
6777        let resolved = runtime
6778            .workspace_for_thread(&ThreadId::from("thread-workflow"))
6779            .await
6780            .unwrap();
6781
6782        assert_eq!(resolved, workspace);
6783    }
6784
6785    #[tokio::test]
6786    async fn automations_can_create_project_thread_with_model_overrides() {
6787        let runtime = Runtime::fake().unwrap();
6788        let workspace = std::env::temp_dir().join("project");
6789        let metadata = runtime
6790            .create_thread_with(CreateThreadRequest {
6791                title: Some("Automation: nightly status".to_string()),
6792                workspace: workspace.display().to_string(),
6793                workspace_id: None,
6794                root_id: None,
6795                provider: Some("mock".to_string()),
6796                model: Some("mock".to_string()),
6797                selection_mode: None,
6798                tool_allowlist: Vec::new(),
6799                developer_instructions: None,
6800                external_tools: Vec::new(),
6801                runner: None,
6802            })
6803            .await
6804            .unwrap();
6805
6806        assert_eq!(
6807            metadata.title.as_deref(),
6808            Some("Automation: nightly status")
6809        );
6810        assert_eq!(metadata.workspace, workspace.display().to_string());
6811        assert_eq!(metadata.provider.as_deref(), Some("mock"));
6812        assert_eq!(metadata.model.as_deref(), Some("mock"));
6813    }
6814
6815    #[tokio::test]
6816    async fn prior_provider_compaction_boundary_is_used_on_next_turn() {
6817        let captured = Arc::new(StdMutex::new(None));
6818        let mut builder = ExtensionRegistryBuilder::new();
6819        builder.inference_engine(Arc::new(CapturingEngine {
6820            request: captured.clone(),
6821        }));
6822        let thread_root = std::env::temp_dir().join(format!(
6823            "roder-provider-compaction-boundary-{}",
6824            uuid::Uuid::new_v4()
6825        ));
6826        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
6827            base_path: thread_root.clone(),
6828        }));
6829        let runtime = Arc::new(
6830            Runtime::new(
6831                builder.build().unwrap(),
6832                RuntimeConfig {
6833                    default_provider: PROVIDER_MOCK.to_string(),
6834                    default_model: "gpt-5.5".to_string(),
6835                    ..RuntimeConfig::default()
6836                },
6837            )
6838            .unwrap(),
6839        );
6840        let thread_id = runtime
6841            .create_thread(Some("Provider compaction boundary".to_string()))
6842            .await
6843            .unwrap()
6844            .thread_id;
6845        let old_turn = "old-turn".to_string();
6846        runtime
6847            .persist_turn_item(
6848                &thread_id,
6849                &old_turn,
6850                &TranscriptItem::UserMessage(UserMessage::text(
6851                    "old history that must not be replayed after provider compaction",
6852                )),
6853            )
6854            .await
6855            .unwrap();
6856        runtime
6857            .persist_turn_item(
6858                &thread_id,
6859                &old_turn,
6860                &TranscriptItem::AssistantMessage(AssistantMessage {
6861                    text: "old answer".to_string(),
6862                    phase: None,
6863                }),
6864            )
6865            .await
6866            .unwrap();
6867        runtime
6868            .persist_turn_item(
6869                &thread_id,
6870                &old_turn,
6871                &TranscriptItem::ProviderMetadata(serde_json::json!({
6872                    "output": [{
6873                        "id": "cmp_1",
6874                        "type": "compaction",
6875                        "encrypted_content": "opaque-state"
6876                    }]
6877                })),
6878            )
6879            .await
6880            .unwrap();
6881        runtime
6882            .persist_turn_item(
6883                &thread_id,
6884                &old_turn,
6885                &TranscriptItem::AssistantMessage(AssistantMessage {
6886                    text: "after compact".to_string(),
6887                    phase: None,
6888                }),
6889            )
6890            .await
6891            .unwrap();
6892
6893        let mut events = runtime.subscribe_events();
6894        runtime
6895            .start_turn(StartTurnRequest {
6896                thread_id: thread_id.clone(),
6897                message: "continue".to_string(),
6898                images: Vec::new(),
6899                provider_override: None,
6900                model_override: None,
6901                reasoning_override: None,
6902                workspace: test_workspace(),
6903                instructions: InstructionBundle::default(),
6904                developer_context: None,
6905                task_ledger_required: false,
6906                service_tier_override: None,
6907            })
6908            .await
6909            .unwrap();
6910        loop {
6911            let envelope = tokio::time::timeout(std::time::Duration::from_secs(5), events.recv())
6912                .await
6913                .unwrap()
6914                .unwrap();
6915            if envelope.thread_id.as_deref() == Some(&thread_id)
6916                && matches!(envelope.event, RoderEvent::TurnCompleted(_))
6917            {
6918                break;
6919            }
6920        }
6921
6922        let request = captured.lock().unwrap().clone().unwrap();
6923        let compaction_idx = request.transcript.iter().position(|item| {
6924            matches!(
6925                item,
6926                TranscriptItem::ProviderMetadata(metadata)
6927                    if crate::compaction::provider_metadata_has_compaction(metadata)
6928            )
6929        });
6930        assert!(
6931            compaction_idx.is_some(),
6932            "next provider request should include the provider compaction item: {:?}",
6933            request.transcript
6934        );
6935        // Skill/context injectors may prepend non-history items, but no prior-turn
6936        // conversation may appear before the latest provider compaction boundary.
6937        let history_before_boundary =
6938            request
6939                .transcript
6940                .iter()
6941                .take(compaction_idx.unwrap())
6942                .any(|item| match item {
6943                    TranscriptItem::AssistantMessage(_)
6944                    | TranscriptItem::ToolCall(_)
6945                    | TranscriptItem::ToolResult(_) => true,
6946                    TranscriptItem::UserMessage(message) => {
6947                        !message.text.contains("<skills>") && message.text != "continue"
6948                    }
6949                    _ => false,
6950                });
6951        assert!(
6952            !history_before_boundary,
6953            "pre-provider-compaction conversation must not be replayed: {:?}",
6954            request.transcript
6955        );
6956        assert!(
6957            !request.transcript.iter().any(|item| {
6958                matches!(
6959                    item,
6960                    TranscriptItem::UserMessage(message)
6961                        if message.text.contains("old history that must not be replayed")
6962                )
6963            }),
6964            "pre-provider-compaction history must not be replayed: {:?}",
6965            request.transcript
6966        );
6967        assert!(
6968            request.transcript.iter().any(|item| {
6969                matches!(
6970                    item,
6971                    TranscriptItem::UserMessage(message) if message.text == "continue"
6972                )
6973            }),
6974            "current continue prompt must be preserved: {:?}",
6975            request.transcript
6976        );
6977        assert_eq!(
6978            request.runtime.auto_compact_token_limit,
6979            Some(945_000),
6980            "gpt-5.5 should keep server-side auto compaction configured"
6981        );
6982
6983        let _ = std::fs::remove_dir_all(thread_root);
6984    }
6985
6986    #[test]
6987    fn server_side_compaction_is_only_enabled_for_supported_models() {
6988        let cfg = RuntimeConfig {
6989            auto_compact_token_limit: Some(123_456),
6990            ..RuntimeConfig::default()
6991        };
6992
6993        assert_eq!(server_side_compaction_threshold(&cfg, "mock"), None);
6994        assert_eq!(
6995            server_side_compaction_threshold(&cfg, "codex-auto-review"),
6996            None
6997        );
6998    }
6999
7000    #[test]
7001    fn reasoning_is_disabled_for_models_without_reasoning_support() {
7002        let cfg = RuntimeConfig {
7003            reasoning: Some(REASONING_HIGH.to_string()),
7004            ..RuntimeConfig::default()
7005        };
7006
7007        assert_eq!(
7008            effective_reasoning_for_model(&cfg, "claude-haiku-4-5-20251001"),
7009            REASONING_NONE
7010        );
7011        assert_eq!(
7012            reasoning_for_model(&cfg, "claude-haiku-4-5-20251001"),
7013            ReasoningConfig::default()
7014        );
7015    }
7016
7017    #[test]
7018    fn unsupported_configured_reasoning_falls_back_to_model_default() {
7019        let cfg = RuntimeConfig {
7020            reasoning: Some(REASONING_MINIMAL.to_string()),
7021            ..RuntimeConfig::default()
7022        };
7023
7024        assert_eq!(
7025            effective_reasoning_for_model(&cfg, "gpt-5.5"),
7026            REASONING_MEDIUM
7027        );
7028    }
7029
7030    #[test]
7031    fn unsupported_configured_gemini_reasoning_is_rejected() {
7032        let mut builder = ExtensionRegistryBuilder::new();
7033        builder.inference_engine(std::sync::Arc::new(FakeInferenceEngine));
7034
7035        let err = match Runtime::new(
7036            builder.build().unwrap(),
7037            RuntimeConfig {
7038                default_model: "gemini-3.5-flash".to_string(),
7039                reasoning: Some(REASONING_XHIGH.to_string()),
7040                ..RuntimeConfig::default()
7041            },
7042        ) {
7043            Ok(_) => panic!("expected unsupported Gemini reasoning to be rejected"),
7044            Err(err) => err,
7045        };
7046
7047        assert!(
7048            err.to_string()
7049                .contains("model gemini-3.5-flash does not support reasoning effort xhigh")
7050        );
7051    }
7052
7053    #[tokio::test]
7054    async fn selecting_none_for_non_reasoning_model_preserves_stored_preference() {
7055        let runtime = Runtime::new(
7056            Runtime::fake().unwrap().registry,
7057            RuntimeConfig {
7058                reasoning: Some(REASONING_HIGH.to_string()),
7059                ..RuntimeConfig::default()
7060            },
7061        )
7062        .unwrap();
7063
7064        let cfg = runtime
7065            .select_provider(
7066                roder_api::catalog::PROVIDER_MOCK.to_string(),
7067                Some("claude-haiku-4-5-20251001".to_string()),
7068                Some(REASONING_NONE.to_string()),
7069            )
7070            .await
7071            .unwrap();
7072
7073        assert_eq!(cfg.reasoning.as_deref(), Some(REASONING_HIGH));
7074        assert_eq!(runtime.effective_reasoning().await, REASONING_NONE);
7075    }
7076
7077    #[tokio::test]
7078    async fn selecting_none_for_model_that_supports_none_updates_preference() {
7079        let runtime = Runtime::new(
7080            Runtime::fake().unwrap().registry,
7081            RuntimeConfig {
7082                reasoning: Some(REASONING_HIGH.to_string()),
7083                ..RuntimeConfig::default()
7084            },
7085        )
7086        .unwrap();
7087
7088        let cfg = runtime
7089            .select_provider(
7090                roder_api::catalog::PROVIDER_MOCK.to_string(),
7091                Some("mock".to_string()),
7092                Some(REASONING_NONE.to_string()),
7093            )
7094            .await
7095            .unwrap();
7096
7097        assert_eq!(cfg.reasoning.as_deref(), Some(REASONING_NONE));
7098    }
7099
7100    #[test]
7101    fn parallel_tool_calls_default_on_with_model_override() {
7102        assert!(parallel_tool_calls_for_model(
7103            &RuntimeConfig::default(),
7104            "custom-model"
7105        ));
7106
7107        let cfg = RuntimeConfig {
7108            model_parallel_tool_calls: std::collections::HashMap::from([(
7109                "custom-model".to_string(),
7110                false,
7111            )]),
7112            ..RuntimeConfig::default()
7113        };
7114
7115        assert!(!parallel_tool_calls_for_model(&cfg, "custom-model"));
7116        assert!(parallel_tool_calls_for_model(&cfg, "other-model"));
7117    }
7118
7119    #[test]
7120    fn profile_parallel_tool_calls_applies_between_config_and_default() {
7121        let cfg = RuntimeConfig {
7122            model_profiles: std::collections::HashMap::from([(
7123                "gpt-5.5".to_string(),
7124                test_model_profile("gpt-5.5"),
7125            )]),
7126            ..RuntimeConfig::default()
7127        };
7128
7129        assert!(!parallel_tool_calls_for_model(&cfg, "gpt-5.5"));
7130
7131        let cfg = RuntimeConfig {
7132            model_parallel_tool_calls: std::collections::HashMap::from([(
7133                "gpt-5.5".to_string(),
7134                true,
7135            )]),
7136            ..cfg
7137        };
7138
7139        assert!(parallel_tool_calls_for_model(&cfg, "gpt-5.5"));
7140    }
7141
7142    struct CapturingEngine {
7143        request: Arc<StdMutex<Option<AgentInferenceRequest>>>,
7144    }
7145
7146    #[async_trait::async_trait]
7147    impl InferenceEngine for CapturingEngine {
7148        fn id(&self) -> String {
7149            roder_api::catalog::PROVIDER_MOCK.to_string()
7150        }
7151
7152        fn capabilities(&self) -> InferenceCapabilities {
7153            InferenceCapabilities::coding_agent_default()
7154        }
7155
7156        async fn list_models(
7157            &self,
7158            _ctx: InferenceProviderContext<'_>,
7159        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7160            Ok(roder_api::catalog::models_for_provider(
7161                roder_api::catalog::PROVIDER_MOCK,
7162                true,
7163            ))
7164        }
7165
7166        async fn stream_turn(
7167            &self,
7168            _ctx: InferenceTurnContext<'_>,
7169            request: AgentInferenceRequest,
7170        ) -> anyhow::Result<InferenceEventStream> {
7171            *self.request.lock().unwrap() = Some(request);
7172            Ok(Box::pin(stream::iter(vec![
7173                Ok(InferenceEvent::MessageDelta(MessageDelta {
7174                    text: "done".to_string(),
7175                    phase: None,
7176                })),
7177                Ok(InferenceEvent::Completed(CompletionMetadata {
7178                    stop_reason: Some("stop".to_string()),
7179                    provider_response_id: None,
7180                })),
7181            ])))
7182        }
7183    }
7184
7185    struct RoutingCaptureEngine {
7186        id: &'static str,
7187        models: Vec<ModelDescriptor>,
7188        requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
7189    }
7190
7191    #[async_trait::async_trait]
7192    impl InferenceEngine for RoutingCaptureEngine {
7193        fn id(&self) -> String {
7194            self.id.to_string()
7195        }
7196
7197        fn capabilities(&self) -> InferenceCapabilities {
7198            InferenceCapabilities::coding_agent_default()
7199        }
7200
7201        async fn list_models(
7202            &self,
7203            _ctx: InferenceProviderContext<'_>,
7204        ) -> anyhow::Result<Vec<ModelDescriptor>> {
7205            Ok(self.models.clone())
7206        }
7207
7208        async fn stream_turn(
7209            &self,
7210            _ctx: InferenceTurnContext<'_>,
7211            request: AgentInferenceRequest,
7212        ) -> anyhow::Result<InferenceEventStream> {
7213            self.requests.lock().unwrap().push(request);
7214            Ok(Box::pin(stream::iter(vec![
7215                Ok(InferenceEvent::MessageDelta(MessageDelta {
7216                    text: "routed".to_string(),
7217                    phase: None,
7218                })),
7219                Ok(InferenceEvent::Completed(CompletionMetadata {
7220                    stop_reason: Some("stop".to_string()),
7221                    provider_response_id: None,
7222                })),
7223            ])))
7224        }
7225    }
7226
7227    struct StaticRouter {
7228        id: &'static str,
7229        decision: InferenceRoutingDecision,
7230        contexts: Arc<StdMutex<Vec<InferenceRoutingContext>>>,
7231    }
7232
7233    #[async_trait::async_trait]
7234    impl InferenceRouter for StaticRouter {
7235        fn id(&self) -> String {
7236            self.id.to_string()
7237        }
7238
7239        async fn route(
7240            &self,
7241            context: InferenceRoutingContext,
7242        ) -> anyhow::Result<InferenceRoutingDecision> {
7243            self.contexts.lock().unwrap().push(context);
7244            Ok(self.decision.clone())
7245        }
7246    }
7247
7248    fn routing_test_model(id: &str, supported_reasoning: &[&str]) -> ModelDescriptor {
7249        ModelDescriptor {
7250            id: id.to_string(),
7251            name: id.to_string(),
7252            context_window: Some(128_000),
7253            default_reasoning: supported_reasoning
7254                .first()
7255                .map(|effort| (*effort).to_string()),
7256            supported_reasoning: supported_reasoning
7257                .iter()
7258                .map(|effort| ReasoningEffortDescriptor {
7259                    effort: (*effort).to_string(),
7260                    description: format!("{effort} reasoning"),
7261                })
7262                .collect(),
7263        }
7264    }
7265
7266    struct TaskLedgerCompletionGateEngine {
7267        calls: StdMutex<u32>,
7268        requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
7269    }
7270
7271    #[async_trait::async_trait]
7272    impl InferenceEngine for TaskLedgerCompletionGateEngine {
7273        fn id(&self) -> String {
7274            roder_api::catalog::PROVIDER_MOCK.to_string()
7275        }
7276
7277        fn capabilities(&self) -> InferenceCapabilities {
7278            InferenceCapabilities::coding_agent_default()
7279        }
7280
7281        async fn list_models(
7282            &self,
7283            _ctx: InferenceProviderContext<'_>,
7284        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7285            Ok(roder_api::catalog::models_for_provider(
7286                roder_api::catalog::PROVIDER_MOCK,
7287                true,
7288            ))
7289        }
7290
7291        async fn stream_turn(
7292            &self,
7293            _ctx: InferenceTurnContext<'_>,
7294            request: AgentInferenceRequest,
7295        ) -> anyhow::Result<InferenceEventStream> {
7296            self.requests.lock().unwrap().push(request);
7297            let mut calls = self.calls.lock().unwrap();
7298            *calls += 1;
7299            let events = match *calls {
7300                1 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7301                    id: "ledger-open".to_string(),
7302                    name: TASK_LEDGER_TOOL_NAME.to_string(),
7303                    arguments: serde_json::json!({
7304                        "tasks": [
7305                            {
7306                                "id": "inspect",
7307                                "content": "Inspect local assets",
7308                                "status": "completed",
7309                                "evidence": "listed workspace"
7310                            },
7311                            {
7312                                "id": "write",
7313                                "content": "Write /app/result.txt",
7314                                "status": "pending"
7315                            }
7316                        ],
7317                        "requireCompletionEvidence": true
7318                    })
7319                    .to_string(),
7320                }))],
7321                3 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7322                    id: "ledger-complete".to_string(),
7323                    name: TASK_LEDGER_TOOL_NAME.to_string(),
7324                    arguments: serde_json::json!({
7325                        "tasks": [
7326                            {
7327                                "id": "inspect",
7328                                "content": "Inspect local assets",
7329                                "status": "completed",
7330                                "evidence": "listed workspace"
7331                            },
7332                            {
7333                                "id": "write",
7334                                "content": "Write /app/result.txt",
7335                                "status": "completed",
7336                                "evidence": "wrote answer"
7337                            }
7338                        ],
7339                        "requireCompletionEvidence": true
7340                    })
7341                    .to_string(),
7342                }))],
7343                _ => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
7344                    text: "final".to_string(),
7345                    phase: None,
7346                }))],
7347            };
7348            Ok(Box::pin(stream::iter(events.into_iter().chain(
7349                std::iter::once(Ok(InferenceEvent::Completed(CompletionMetadata {
7350                    stop_reason: Some("stop".to_string()),
7351                    provider_response_id: None,
7352                }))),
7353            ))))
7354        }
7355    }
7356
7357    struct VerificationGateEngine {
7358        calls: StdMutex<u32>,
7359    }
7360
7361    #[async_trait::async_trait]
7362    impl InferenceEngine for VerificationGateEngine {
7363        fn id(&self) -> String {
7364            roder_api::catalog::PROVIDER_MOCK.to_string()
7365        }
7366
7367        fn capabilities(&self) -> InferenceCapabilities {
7368            InferenceCapabilities::coding_agent_default()
7369        }
7370
7371        async fn list_models(
7372            &self,
7373            _ctx: InferenceProviderContext<'_>,
7374        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7375            Ok(roder_api::catalog::models_for_provider(
7376                roder_api::catalog::PROVIDER_MOCK,
7377                true,
7378            ))
7379        }
7380
7381        async fn stream_turn(
7382            &self,
7383            _ctx: InferenceTurnContext<'_>,
7384            request: AgentInferenceRequest,
7385        ) -> anyhow::Result<InferenceEventStream> {
7386            let mut calls = self.calls.lock().unwrap();
7387            *calls += 1;
7388            let events = match *calls {
7389                1 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7390                    id: "write-1".to_string(),
7391                    name: "write_file".to_string(),
7392                    arguments: serde_json::json!({
7393                        "path": "src/lib.rs",
7394                        "content": "pub fn answer() -> u8 { 42 }\n"
7395                    })
7396                    .to_string(),
7397                }))],
7398                2 => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
7399                    text: "done too early".to_string(),
7400                    phase: None,
7401                }))],
7402                3 if request.transcript.iter().any(|item| {
7403                    matches!(
7404                        item,
7405                        TranscriptItem::UserMessage(message)
7406                            if message.text.contains("Verification gate blocked final completion")
7407                    )
7408                }) =>
7409                {
7410                    vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7411                        id: "verify-1".to_string(),
7412                        name: crate::verification_gate::VERIFICATION_TOOL_NAME.to_string(),
7413                        arguments: serde_json::json!({
7414                            "originalTask": "write code",
7415                            "changedFiles": ["src/lib.rs"],
7416                            "toolEvidence": ["write_file wrote src/lib.rs"],
7417                            "testsRun": ["cargo test -p roder-core verification_gate"],
7418                            "openGaps": [],
7419                            "status": "completed"
7420                        })
7421                        .to_string(),
7422                    }))]
7423                }
7424                _ => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
7425                    text: "verified final".to_string(),
7426                    phase: None,
7427                }))],
7428            };
7429            Ok(Box::pin(stream::iter(events.into_iter().chain(
7430                std::iter::once(Ok(InferenceEvent::Completed(CompletionMetadata {
7431                    stop_reason: Some("stop".to_string()),
7432                    provider_response_id: None,
7433                }))),
7434            ))))
7435        }
7436    }
7437
7438    struct SpeedPolicyEngine {
7439        calls: StdMutex<u32>,
7440        requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
7441    }
7442
7443    #[async_trait::async_trait]
7444    impl InferenceEngine for SpeedPolicyEngine {
7445        fn id(&self) -> String {
7446            roder_api::catalog::PROVIDER_MOCK.to_string()
7447        }
7448
7449        fn capabilities(&self) -> InferenceCapabilities {
7450            InferenceCapabilities::coding_agent_default()
7451        }
7452
7453        async fn list_models(
7454            &self,
7455            _ctx: InferenceProviderContext<'_>,
7456        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7457            Ok(roder_api::catalog::models_for_provider(
7458                roder_api::catalog::PROVIDER_MOCK,
7459                true,
7460            ))
7461        }
7462
7463        async fn stream_turn(
7464            &self,
7465            _ctx: InferenceTurnContext<'_>,
7466            request: AgentInferenceRequest,
7467        ) -> anyhow::Result<InferenceEventStream> {
7468            self.requests.lock().unwrap().push(request.clone());
7469            let mut calls = self.calls.lock().unwrap();
7470            *calls += 1;
7471            let events = match *calls {
7472                1 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7473                    id: "write-1".to_string(),
7474                    name: "write_file".to_string(),
7475                    arguments: serde_json::json!({
7476                        "path": "src/lib.rs",
7477                        "content": "pub fn answer() -> u8 { 42 }\n"
7478                    })
7479                    .to_string(),
7480                }))],
7481                3 if request.transcript.iter().any(|item| {
7482                    matches!(
7483                        item,
7484                        TranscriptItem::UserMessage(message)
7485                            if message.text.contains("Verification gate blocked final completion")
7486                    )
7487                }) =>
7488                {
7489                    vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7490                        id: "verify-1".to_string(),
7491                        name: crate::verification_gate::VERIFICATION_TOOL_NAME.to_string(),
7492                        arguments: serde_json::json!({
7493                            "originalTask": "write code",
7494                            "changedFiles": ["src/lib.rs"],
7495                            "toolEvidence": ["write_file wrote src/lib.rs"],
7496                            "testsRun": ["cargo test -p roder-core speed_policy"],
7497                            "openGaps": [],
7498                            "status": "completed"
7499                        })
7500                        .to_string(),
7501                    }))]
7502                }
7503                _ => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
7504                    text: "done".to_string(),
7505                    phase: None,
7506                }))],
7507            };
7508            Ok(Box::pin(stream::iter(events.into_iter().chain(
7509                std::iter::once(Ok(InferenceEvent::Completed(CompletionMetadata {
7510                    stop_reason: Some("stop".to_string()),
7511                    provider_response_id: None,
7512                }))),
7513            ))))
7514        }
7515    }
7516
7517    struct SwitchCaptureEngine {
7518        requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
7519    }
7520
7521    #[async_trait::async_trait]
7522    impl InferenceEngine for SwitchCaptureEngine {
7523        fn id(&self) -> String {
7524            roder_api::catalog::PROVIDER_MOCK.to_string()
7525        }
7526
7527        fn capabilities(&self) -> InferenceCapabilities {
7528            InferenceCapabilities::coding_agent_default()
7529        }
7530
7531        async fn list_models(
7532            &self,
7533            _ctx: InferenceProviderContext<'_>,
7534        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7535            Ok(roder_api::catalog::models_for_provider(
7536                roder_api::catalog::PROVIDER_MOCK,
7537                true,
7538            ))
7539        }
7540
7541        async fn stream_turn(
7542            &self,
7543            _ctx: InferenceTurnContext<'_>,
7544            request: AgentInferenceRequest,
7545        ) -> anyhow::Result<InferenceEventStream> {
7546            self.requests.lock().unwrap().push(request);
7547            Ok(Box::pin(stream::iter(vec![
7548                Ok(InferenceEvent::MessageDelta(MessageDelta {
7549                    text: "done".to_string(),
7550                    phase: None,
7551                })),
7552                Ok(InferenceEvent::Completed(CompletionMetadata {
7553                    stop_reason: Some("stop".to_string()),
7554                    provider_response_id: None,
7555                })),
7556            ])))
7557        }
7558    }
7559
7560    struct DeadlineEngine;
7561
7562    #[async_trait::async_trait]
7563    impl InferenceEngine for DeadlineEngine {
7564        fn id(&self) -> String {
7565            roder_api::catalog::PROVIDER_MOCK.to_string()
7566        }
7567
7568        fn capabilities(&self) -> InferenceCapabilities {
7569            InferenceCapabilities::coding_agent_default()
7570        }
7571
7572        async fn list_models(
7573            &self,
7574            _ctx: InferenceProviderContext<'_>,
7575        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7576            Ok(Vec::new())
7577        }
7578
7579        async fn stream_turn(
7580            &self,
7581            _ctx: InferenceTurnContext<'_>,
7582            _request: AgentInferenceRequest,
7583        ) -> anyhow::Result<InferenceEventStream> {
7584            Ok(Box::pin(stream::once(async {
7585                tokio::time::sleep(std::time::Duration::from_secs(60)).await;
7586                Ok(InferenceEvent::MessageDelta(MessageDelta {
7587                    text: "too late".to_string(),
7588                    phase: None,
7589                }))
7590            })))
7591        }
7592    }
7593
7594    struct WriteFileContributor;
7595
7596    impl ToolContributor for WriteFileContributor {
7597        fn id(&self) -> String {
7598            "test-write".to_string()
7599        }
7600
7601        fn contribute(&self, registry: &mut ToolRegistry) -> anyhow::Result<()> {
7602            registry.register(Arc::new(WriteFileTool))
7603        }
7604    }
7605
7606    struct WriteFileTool;
7607
7608    #[async_trait::async_trait]
7609    impl ToolExecutor for WriteFileTool {
7610        fn spec(&self) -> ToolSpec {
7611            ToolSpec {
7612                name: "write_file".to_string(),
7613                description: "Write a test file.".to_string(),
7614                parameters: serde_json::json!({
7615                    "type": "object",
7616                    "properties": {
7617                        "path": { "type": "string" },
7618                        "content": { "type": "string" }
7619                    },
7620                    "required": ["path", "content"],
7621                    "additionalProperties": false
7622                }),
7623            }
7624        }
7625
7626        async fn execute(
7627            &self,
7628            _ctx: ToolExecutionContext,
7629            call: ToolCall,
7630        ) -> anyhow::Result<ToolResult> {
7631            let path = call
7632                .arguments
7633                .get("path")
7634                .and_then(serde_json::Value::as_str)
7635                .unwrap_or("src/lib.rs");
7636            Ok(ToolResult {
7637                id: call.id,
7638                name: call.name,
7639                text: format!("wrote {path}"),
7640                data: serde_json::json!({ "path": path }),
7641                is_error: false,
7642            })
7643        }
7644    }
7645
7646    struct ProfileToolContributor;
7647
7648    impl ToolContributor for ProfileToolContributor {
7649        fn id(&self) -> String {
7650            "profile-tools".to_string()
7651        }
7652
7653        fn contribute(&self, registry: &mut ToolRegistry) -> anyhow::Result<()> {
7654            for name in ["apply_patch", "edit", "multi_edit", "write_file"] {
7655                registry.register(Arc::new(ProfileTool {
7656                    name: name.to_string(),
7657                }))?;
7658            }
7659            Ok(())
7660        }
7661    }
7662
7663    struct ProfileTool {
7664        name: String,
7665    }
7666
7667    #[async_trait::async_trait]
7668    impl ToolExecutor for ProfileTool {
7669        fn spec(&self) -> ToolSpec {
7670            ToolSpec {
7671                name: self.name.clone(),
7672                description: format!("{} test tool", self.name),
7673                parameters: serde_json::json!({
7674                    "type": "object",
7675                    "properties": {
7676                        "path": { "type": "string" },
7677                        "content": { "type": "string" }
7678                    },
7679                    "required": ["path", "content"],
7680                    "additionalProperties": false
7681                }),
7682            }
7683        }
7684
7685        async fn execute(
7686            &self,
7687            _ctx: ToolExecutionContext,
7688            call: ToolCall,
7689        ) -> anyhow::Result<ToolResult> {
7690            Ok(ToolResult {
7691                id: call.id,
7692                name: call.name,
7693                text: "ok".to_string(),
7694                data: serde_json::json!({}),
7695                is_error: false,
7696            })
7697        }
7698    }
7699
7700    fn test_model_profile(model: &str) -> ModelHarnessProfile {
7701        ModelHarnessProfile {
7702            model: model.to_string(),
7703            provider: roder_api::catalog::PROVIDER_OPENAI.to_string(),
7704            provider_family: ProviderFamily::OpenAi,
7705            edit_tool: Some(EDIT_TOOL_EDIT.to_string()),
7706            schema_policy: ModelSchemaPolicy::StandardRequiredFirst,
7707            instruction_overlay: ModelInstructionOverlay::IntuitiveContext,
7708            reasoning: ModelProfileReasoning {
7709                orientation: Some(REASONING_LOW.to_string()),
7710                execution: Some(REASONING_LOW.to_string()),
7711                verification: Some(REASONING_LOW.to_string()),
7712                recovery: Some(REASONING_LOW.to_string()),
7713            },
7714            parallel_tool_calls: Some(false),
7715            auto_compact_token_limit: Some(123_000),
7716        }
7717    }
7718
7719    async fn captured_profile_request(cfg: RuntimeConfig) -> AgentInferenceRequest {
7720        let captured = Arc::new(StdMutex::new(None));
7721        let mut builder = ExtensionRegistryBuilder::new();
7722        builder.inference_engine(Arc::new(CapturingEngine {
7723            request: captured.clone(),
7724        }));
7725        builder.tool_contributor(Arc::new(ProfileToolContributor));
7726        let runtime = Arc::new(Runtime::new(builder.build().unwrap(), cfg).unwrap());
7727        let mut rx = runtime.subscribe_events();
7728        let turn_id = runtime
7729            .start_turn(StartTurnRequest {
7730                thread_id: "thread-model-profile".to_string(),
7731                message: "use profile knobs".to_string(),
7732                images: Vec::new(),
7733                provider_override: None,
7734                model_override: None,
7735                reasoning_override: None,
7736                workspace: test_workspace(),
7737                instructions: InstructionBundle {
7738                    system: None,
7739                    developer: Some("base developer".to_string()),
7740                    developer_context: None,
7741                },
7742                developer_context: None,
7743                task_ledger_required: false,
7744                service_tier_override: None,
7745            })
7746            .await
7747            .unwrap();
7748
7749        tokio::time::timeout(std::time::Duration::from_secs(5), async {
7750            loop {
7751                let envelope = rx.recv().await.unwrap();
7752                if envelope.turn_id.as_deref() != Some(&turn_id) {
7753                    continue;
7754                }
7755                match envelope.event {
7756                    RoderEvent::TurnCompleted(_) => break,
7757                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7758                    _ => {}
7759                }
7760            }
7761        })
7762        .await
7763        .unwrap();
7764
7765        captured.lock().unwrap().clone().unwrap()
7766    }
7767
7768    struct ToolThenStopEngine {
7769        calls: StdMutex<u32>,
7770    }
7771
7772    #[async_trait::async_trait]
7773    impl InferenceEngine for ToolThenStopEngine {
7774        fn id(&self) -> String {
7775            roder_api::catalog::PROVIDER_MOCK.to_string()
7776        }
7777
7778        fn capabilities(&self) -> InferenceCapabilities {
7779            InferenceCapabilities::coding_agent_default()
7780        }
7781
7782        async fn list_models(
7783            &self,
7784            _ctx: InferenceProviderContext<'_>,
7785        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7786            Ok(roder_api::catalog::models_for_provider(
7787                roder_api::catalog::PROVIDER_MOCK,
7788                true,
7789            ))
7790        }
7791
7792        async fn stream_turn(
7793            &self,
7794            _ctx: InferenceTurnContext<'_>,
7795            _request: AgentInferenceRequest,
7796        ) -> anyhow::Result<InferenceEventStream> {
7797            let mut calls = self.calls.lock().unwrap();
7798            *calls += 1;
7799            let events = match *calls {
7800                1 => vec![
7801                    Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7802                        id: "write-1".to_string(),
7803                        name: "write_file".to_string(),
7804                        arguments: serde_json::json!({
7805                            "path": "src/lib.rs",
7806                            "content": "pub fn answer() -> u8 { 42 }\n"
7807                        })
7808                        .to_string(),
7809                    })),
7810                    Ok(InferenceEvent::Completed(CompletionMetadata {
7811                        stop_reason: Some("tool_use".to_string()),
7812                        provider_response_id: None,
7813                    })),
7814                ],
7815                _ => vec![
7816                    Ok(InferenceEvent::MessageDelta(MessageDelta {
7817                        text: "final".to_string(),
7818                        phase: None,
7819                    })),
7820                    Ok(InferenceEvent::Completed(CompletionMetadata {
7821                        stop_reason: Some("end_turn".to_string()),
7822                        provider_response_id: None,
7823                    })),
7824                ],
7825            };
7826            Ok(Box::pin(stream::iter(events)))
7827        }
7828    }
7829
7830    #[tokio::test]
7831    async fn turn_completed_reports_terminal_step_finish_reason() {
7832        let mut builder = ExtensionRegistryBuilder::new();
7833        builder.inference_engine(Arc::new(ToolThenStopEngine {
7834            calls: StdMutex::new(0),
7835        }));
7836        builder.tool_contributor(Arc::new(WriteFileContributor));
7837        let runtime = Arc::new(
7838            Runtime::new(
7839                builder.build().unwrap(),
7840                RuntimeConfig {
7841                    policy_mode: PolicyMode::Bypass,
7842                    agent_swarm_mode: false,
7843                    ultra_mode: false,
7844                    ..RuntimeConfig::default()
7845                },
7846            )
7847            .unwrap(),
7848        );
7849        let mut rx = runtime.subscribe_events();
7850        let turn_id = runtime
7851            .start_turn(StartTurnRequest {
7852                thread_id: "thread-finish-reason".to_string(),
7853                message: "write then finish".to_string(),
7854                images: Vec::new(),
7855                provider_override: None,
7856                model_override: None,
7857                reasoning_override: None,
7858                workspace: test_workspace(),
7859                instructions: InstructionBundle {
7860                    system: None,
7861                    developer: None,
7862                    developer_context: None,
7863                },
7864                developer_context: None,
7865                task_ledger_required: false,
7866                service_tier_override: None,
7867            })
7868            .await
7869            .unwrap();
7870
7871        let completed = tokio::time::timeout(std::time::Duration::from_secs(5), async {
7872            loop {
7873                let envelope = rx.recv().await.unwrap();
7874                if envelope.turn_id.as_deref() != Some(&turn_id) {
7875                    continue;
7876                }
7877                match envelope.event {
7878                    RoderEvent::TurnCompleted(event) => break event,
7879                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7880                    _ => {}
7881                }
7882            }
7883        })
7884        .await
7885        .unwrap();
7886
7887        // The mid-turn tool_use stop reason must not leak; the terminal
7888        // end_turn step decides the turn's finish reason.
7889        assert_eq!(completed.finish_reason.as_deref(), Some("stop"));
7890    }
7891
7892    #[tokio::test]
7893    async fn inference_router_selection_changes_request_model_and_records_event() {
7894        let default_requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7895        let routed_requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7896        let contexts = Arc::new(StdMutex::new(Vec::<InferenceRoutingContext>::new()));
7897        let selected = ModelSelection {
7898            provider: "routed-provider".to_string(),
7899            model: "routed-model".to_string(),
7900        };
7901        let default = ModelSelection {
7902            provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7903            model: "mock".to_string(),
7904        };
7905        let decision = InferenceRoutingDecision {
7906            reasoning: Some(ReasoningConfig {
7907                enabled: true,
7908                level: Some(REASONING_LOW.to_string()),
7909            }),
7910            confidence: Some(0.91),
7911            baseline: Some(default.clone()),
7912            matched_signals: vec![roder_api::inference_routing::InferenceRoutingSignal::new(
7913                "intent", "routine",
7914            )],
7915            ..InferenceRoutingDecision::selected("test-router", selected.clone(), "routine request")
7916        };
7917
7918        let mut builder = ExtensionRegistryBuilder::new();
7919        builder.inference_engine(Arc::new(RoutingCaptureEngine {
7920            id: roder_api::catalog::PROVIDER_MOCK,
7921            models: vec![routing_test_model("mock", &[REASONING_LOW])],
7922            requests: default_requests.clone(),
7923        }));
7924        builder.inference_engine(Arc::new(RoutingCaptureEngine {
7925            id: "routed-provider",
7926            models: vec![routing_test_model(
7927                "routed-model",
7928                &[REASONING_LOW, REASONING_MEDIUM],
7929            )],
7930            requests: routed_requests.clone(),
7931        }));
7932        builder.inference_router(Arc::new(StaticRouter {
7933            id: "test-router",
7934            decision,
7935            contexts: contexts.clone(),
7936        }));
7937        let thread_root =
7938            std::env::temp_dir().join(format!("roder-routing-auto-{}", uuid::Uuid::new_v4()));
7939        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
7940            base_path: thread_root.clone(),
7941        }));
7942        let runtime = Arc::new(
7943            Runtime::new(
7944                builder.build().unwrap(),
7945                RuntimeConfig {
7946                    default_provider: default.provider.clone(),
7947                    default_model: default.model.clone(),
7948                    ..RuntimeConfig::default()
7949                },
7950            )
7951            .unwrap(),
7952        );
7953        let thread_id = runtime
7954            .create_thread_with(CreateThreadRequest {
7955                title: Some("Routing auto".to_string()),
7956                workspace: test_workspace(),
7957                workspace_id: None,
7958                root_id: None,
7959                provider: Some(default.provider.clone()),
7960                model: Some(default.model.clone()),
7961                tool_allowlist: Vec::new(),
7962                developer_instructions: None,
7963                external_tools: Vec::new(),
7964                selection_mode: Some(ModelSelectionMode::auto(
7965                    "test-router:coding",
7966                    "test-router",
7967                    "Auto: Coding",
7968                    default.clone(),
7969                    Some("coding".to_string()),
7970                    None,
7971                )),
7972                runner: None,
7973            })
7974            .await
7975            .unwrap()
7976            .thread_id;
7977        let mut rx = runtime.subscribe_events();
7978        let turn_id = runtime
7979            .start_turn(StartTurnRequest {
7980                thread_id: thread_id.clone(),
7981                message: "small cleanup".to_string(),
7982                images: Vec::new(),
7983                provider_override: None,
7984                model_override: None,
7985                reasoning_override: None,
7986                workspace: test_workspace(),
7987                instructions: InstructionBundle::default(),
7988                developer_context: None,
7989                task_ledger_required: false,
7990                service_tier_override: None,
7991            })
7992            .await
7993            .unwrap();
7994
7995        let mut routing_event = None;
7996        let mut inference_started = None;
7997        tokio::time::timeout(std::time::Duration::from_secs(5), async {
7998            loop {
7999                let envelope = rx.recv().await.unwrap();
8000                if envelope.turn_id.as_deref() != Some(&turn_id) {
8001                    continue;
8002                }
8003                match envelope.event {
8004                    RoderEvent::InferenceRoutingDecision(event) => {
8005                        routing_event = Some(event);
8006                    }
8007                    RoderEvent::InferenceStarted(event) => {
8008                        inference_started = Some(event);
8009                    }
8010                    RoderEvent::TurnCompleted(_) => break,
8011                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8012                    _ => {}
8013                }
8014            }
8015        })
8016        .await
8017        .unwrap();
8018
8019        assert!(default_requests.lock().unwrap().is_empty());
8020        let routed_requests = routed_requests.lock().unwrap();
8021        assert_eq!(routed_requests.len(), 1);
8022        assert_eq!(routed_requests[0].model, selected);
8023        assert_eq!(
8024            routed_requests[0].reasoning.level.as_deref(),
8025            Some(REASONING_LOW)
8026        );
8027        assert_eq!(
8028            routed_requests[0].metadata["inferenceRouting"]["outcome"],
8029            "selected"
8030        );
8031
8032        let routing_event = routing_event.expect("routing decision event");
8033        assert_eq!(routing_event.default_selection, default);
8034        assert_eq!(routing_event.selected_selection, selected);
8035        assert_eq!(
8036            routing_event.decision.outcome,
8037            InferenceRoutingOutcome::Selected
8038        );
8039        assert_eq!(
8040            inference_started.expect("inference started event").model,
8041            selected
8042        );
8043
8044        let contexts = contexts.lock().unwrap();
8045        assert_eq!(contexts.len(), 1);
8046        assert_eq!(contexts[0].default_selection, default);
8047        assert_eq!(contexts[0].candidates.len(), 2);
8048        assert!(
8049            contexts[0]
8050                .signals
8051                .iter()
8052                .any(|signal| signal.key == "profile" && signal.value == "coding")
8053        );
8054        let _ = std::fs::remove_dir_all(thread_root);
8055    }
8056
8057    #[tokio::test]
8058    async fn inference_router_is_bypassed_for_explicit_selection() {
8059        let requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
8060        let contexts = Arc::new(StdMutex::new(Vec::<InferenceRoutingContext>::new()));
8061        let selected = ModelSelection {
8062            provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8063            model: "mock".to_string(),
8064        };
8065
8066        let mut builder = ExtensionRegistryBuilder::new();
8067        builder.inference_engine(Arc::new(RoutingCaptureEngine {
8068            id: roder_api::catalog::PROVIDER_MOCK,
8069            models: vec![routing_test_model("mock", &[REASONING_LOW])],
8070            requests: requests.clone(),
8071        }));
8072        builder.inference_router(Arc::new(StaticRouter {
8073            id: "test-router",
8074            decision: InferenceRoutingDecision::selected(
8075                "test-router",
8076                ModelSelection {
8077                    provider: "missing".to_string(),
8078                    model: "missing".to_string(),
8079                },
8080                "would route if called",
8081            ),
8082            contexts: contexts.clone(),
8083        }));
8084        let thread_root =
8085            std::env::temp_dir().join(format!("roder-routing-explicit-{}", uuid::Uuid::new_v4()));
8086        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8087            base_path: thread_root.clone(),
8088        }));
8089        let runtime = Arc::new(
8090            Runtime::new(
8091                builder.build().unwrap(),
8092                RuntimeConfig {
8093                    default_provider: selected.provider.clone(),
8094                    default_model: selected.model.clone(),
8095                    inference_router: RuntimeInferenceRouterConfig {
8096                        enabled: true,
8097                        router_id: Some("test-router".to_string()),
8098                    },
8099                    ..RuntimeConfig::default()
8100                },
8101            )
8102            .unwrap(),
8103        );
8104        let thread_id = runtime
8105            .create_thread_with(CreateThreadRequest {
8106                title: Some("Routing explicit".to_string()),
8107                workspace: test_workspace(),
8108                workspace_id: None,
8109                root_id: None,
8110                provider: Some(selected.provider.clone()),
8111                model: Some(selected.model.clone()),
8112                tool_allowlist: Vec::new(),
8113                developer_instructions: None,
8114                external_tools: Vec::new(),
8115                selection_mode: Some(ModelSelectionMode::auto(
8116                    "test-router:default",
8117                    "test-router",
8118                    "Auto",
8119                    selected.clone(),
8120                    None,
8121                    None,
8122                )),
8123                runner: None,
8124            })
8125            .await
8126            .unwrap()
8127            .thread_id;
8128        let mut rx = runtime.subscribe_events();
8129        let turn_id = runtime
8130            .start_turn(StartTurnRequest {
8131                thread_id,
8132                message: "use explicit selection".to_string(),
8133                images: Vec::new(),
8134                provider_override: Some(selected.provider.clone()),
8135                model_override: Some(selected.model.clone()),
8136                reasoning_override: None,
8137                workspace: test_workspace(),
8138                instructions: InstructionBundle::default(),
8139                developer_context: None,
8140                task_ledger_required: false,
8141                service_tier_override: None,
8142            })
8143            .await
8144            .unwrap();
8145
8146        let mut saw_routing_event = false;
8147        tokio::time::timeout(std::time::Duration::from_secs(5), async {
8148            loop {
8149                let envelope = rx.recv().await.unwrap();
8150                if envelope.turn_id.as_deref() != Some(&turn_id) {
8151                    continue;
8152                }
8153                match envelope.event {
8154                    RoderEvent::InferenceRoutingDecision(_) => {
8155                        saw_routing_event = true;
8156                    }
8157                    RoderEvent::TurnCompleted(_) => break,
8158                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8159                    _ => {}
8160                }
8161            }
8162        })
8163        .await
8164        .unwrap();
8165
8166        assert!(!saw_routing_event);
8167        assert!(contexts.lock().unwrap().is_empty());
8168        let requests = requests.lock().unwrap();
8169        assert_eq!(requests.len(), 1);
8170        assert_eq!(requests[0].model, selected);
8171        let _ = std::fs::remove_dir_all(thread_root);
8172    }
8173
8174    #[tokio::test]
8175    async fn inference_router_is_bypassed_for_manual_selection_mode() {
8176        let requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
8177        let contexts = Arc::new(StdMutex::new(Vec::<InferenceRoutingContext>::new()));
8178        let selected = ModelSelection {
8179            provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8180            model: "mock".to_string(),
8181        };
8182
8183        let mut builder = ExtensionRegistryBuilder::new();
8184        builder.inference_engine(Arc::new(RoutingCaptureEngine {
8185            id: roder_api::catalog::PROVIDER_MOCK,
8186            models: vec![routing_test_model("mock", &[REASONING_LOW])],
8187            requests: requests.clone(),
8188        }));
8189        builder.inference_router(Arc::new(StaticRouter {
8190            id: "test-router",
8191            decision: InferenceRoutingDecision::selected(
8192                "test-router",
8193                ModelSelection {
8194                    provider: "missing".to_string(),
8195                    model: "missing".to_string(),
8196                },
8197                "would route if called",
8198            ),
8199            contexts: contexts.clone(),
8200        }));
8201        let thread_root =
8202            std::env::temp_dir().join(format!("roder-routing-manual-{}", uuid::Uuid::new_v4()));
8203        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8204            base_path: thread_root.clone(),
8205        }));
8206        let runtime = Arc::new(
8207            Runtime::new(
8208                builder.build().unwrap(),
8209                RuntimeConfig {
8210                    default_provider: selected.provider.clone(),
8211                    default_model: selected.model.clone(),
8212                    inference_router: RuntimeInferenceRouterConfig {
8213                        enabled: true,
8214                        router_id: Some("test-router".to_string()),
8215                    },
8216                    ..RuntimeConfig::default()
8217                },
8218            )
8219            .unwrap(),
8220        );
8221        let thread_id = runtime
8222            .create_thread_with(CreateThreadRequest {
8223                title: Some("Routing manual".to_string()),
8224                workspace: test_workspace(),
8225                workspace_id: None,
8226                root_id: None,
8227                provider: Some(selected.provider.clone()),
8228                model: Some(selected.model.clone()),
8229                tool_allowlist: Vec::new(),
8230                developer_instructions: None,
8231                external_tools: Vec::new(),
8232                selection_mode: Some(ModelSelectionMode::manual(
8233                    selected.provider.clone(),
8234                    selected.model.clone(),
8235                    None,
8236                )),
8237                runner: None,
8238            })
8239            .await
8240            .unwrap()
8241            .thread_id;
8242        let mut rx = runtime.subscribe_events();
8243        let turn_id = runtime
8244            .start_turn(StartTurnRequest {
8245                thread_id,
8246                message: "use selected manual model".to_string(),
8247                images: Vec::new(),
8248                provider_override: None,
8249                model_override: None,
8250                reasoning_override: None,
8251                workspace: test_workspace(),
8252                instructions: InstructionBundle::default(),
8253                developer_context: None,
8254                task_ledger_required: false,
8255                service_tier_override: None,
8256            })
8257            .await
8258            .unwrap();
8259
8260        let mut saw_routing_event = false;
8261        tokio::time::timeout(std::time::Duration::from_secs(5), async {
8262            loop {
8263                let envelope = rx.recv().await.unwrap();
8264                if envelope.turn_id.as_deref() != Some(&turn_id) {
8265                    continue;
8266                }
8267                match envelope.event {
8268                    RoderEvent::InferenceRoutingDecision(_) => {
8269                        saw_routing_event = true;
8270                    }
8271                    RoderEvent::TurnCompleted(_) => break,
8272                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8273                    _ => {}
8274                }
8275            }
8276        })
8277        .await
8278        .unwrap();
8279
8280        assert!(!saw_routing_event);
8281        assert!(contexts.lock().unwrap().is_empty());
8282        let requests = requests.lock().unwrap();
8283        assert_eq!(requests.len(), 1);
8284        assert_eq!(requests[0].model, selected);
8285        let _ = std::fs::remove_dir_all(thread_root);
8286    }
8287
8288    #[test]
8289    fn enabled_inference_router_requires_registered_router() {
8290        let mut builder = ExtensionRegistryBuilder::new();
8291        builder.inference_engine(Arc::new(FakeInferenceEngine));
8292
8293        let err = match Runtime::new(
8294            builder.build().unwrap(),
8295            RuntimeConfig {
8296                inference_router: RuntimeInferenceRouterConfig {
8297                    enabled: true,
8298                    router_id: Some("missing-router".to_string()),
8299                },
8300                ..RuntimeConfig::default()
8301            },
8302        ) {
8303            Ok(_) => panic!("runtime should reject unknown inference router"),
8304            Err(err) => err,
8305        };
8306
8307        assert!(
8308            err.to_string()
8309                .contains("inference router \"missing-router\" is not registered")
8310        );
8311    }
8312
8313    #[tokio::test]
8314    async fn model_profile_routes_request_knobs_to_next_inference() {
8315        let request = captured_profile_request(RuntimeConfig {
8316            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8317            default_model: "gpt-5.5".to_string(),
8318            model_profiles: std::collections::HashMap::from([(
8319                "gpt-5.5".to_string(),
8320                test_model_profile("gpt-5.5"),
8321            )]),
8322            ..RuntimeConfig::default()
8323        })
8324        .await;
8325
8326        let tool_names = request
8327            .tools
8328            .iter()
8329            .map(|tool| tool.name.as_str())
8330            .collect::<Vec<_>>();
8331        assert!(tool_names.contains(&"apply_patch"));
8332        assert!(tool_names.contains(&"edit"));
8333        assert!(tool_names.contains(&"multi_edit"));
8334        assert!(tool_names.contains(&"write_file"));
8335        assert_eq!(request.reasoning.level.as_deref(), Some(REASONING_LOW));
8336        assert_eq!(request.runtime.parallel_tool_calls, Some(false));
8337        assert_eq!(request.runtime.auto_compact_token_limit, Some(123_000));
8338        assert!(
8339            request
8340                .instructions
8341                .developer
8342                .as_deref()
8343                .unwrap_or_default()
8344                .contains("Use the provided context as the current working set")
8345        );
8346        assert_eq!(
8347            request
8348                .metadata
8349                .pointer("/modelProfile/schemaPolicy")
8350                .and_then(serde_json::Value::as_str),
8351            Some("standard_required_first")
8352        );
8353    }
8354
8355    #[tokio::test]
8356    async fn ultra_reasoning_reaches_transport_state_and_enables_proactive_delegation() {
8357        let request = captured_profile_request(RuntimeConfig {
8358            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8359            default_model: "gpt-5.6-sol".to_string(),
8360            reasoning: Some(REASONING_ULTRA.to_string()),
8361            ..RuntimeConfig::default()
8362        })
8363        .await;
8364
8365        assert_eq!(request.reasoning.level.as_deref(), Some(REASONING_ULTRA));
8366        let developer = request
8367            .instructions
8368            .developer
8369            .as_deref()
8370            .expect("ultra developer instructions");
8371        assert!(developer.contains("Proactive multi-agent delegation is active"));
8372    }
8373
8374    #[tokio::test]
8375    async fn lower_sol_effort_requires_explicit_multi_agent_request() {
8376        let request = captured_profile_request(RuntimeConfig {
8377            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8378            default_model: "gpt-5.6-sol".to_string(),
8379            reasoning: Some(roder_api::catalog::REASONING_MEDIUM.to_string()),
8380            ..RuntimeConfig::default()
8381        })
8382        .await;
8383
8384        let developer = request
8385            .instructions
8386            .developer
8387            .as_deref()
8388            .expect("sol developer instructions");
8389        assert!(developer.contains("Do not spawn sub-agents unless"));
8390        assert!(!developer.contains("Proactive multi-agent delegation is active"));
8391    }
8392
8393    #[tokio::test]
8394    async fn luna_does_not_receive_codex_v2_multi_agent_policy() {
8395        let request = captured_profile_request(RuntimeConfig {
8396            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8397            default_model: "gpt-5.6-luna".to_string(),
8398            reasoning: Some(roder_api::catalog::REASONING_MAX.to_string()),
8399            ..RuntimeConfig::default()
8400        })
8401        .await;
8402
8403        let developer = request
8404            .instructions
8405            .developer
8406            .as_deref()
8407            .unwrap_or_default();
8408        assert!(!developer.contains("Do not spawn sub-agents unless"));
8409        assert!(!developer.contains("Proactive multi-agent delegation is active"));
8410    }
8411
8412    #[tokio::test]
8413    async fn ultra_mode_enables_proactive_multi_agent_for_any_model() {
8414        // Luna has no Ultra effort, but ultra mode still injects proactive policy.
8415        let request = captured_profile_request(RuntimeConfig {
8416            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8417            default_model: "gpt-5.6-luna".to_string(),
8418            reasoning: Some(roder_api::catalog::REASONING_MAX.to_string()),
8419            ultra_mode: true,
8420            ..RuntimeConfig::default()
8421        })
8422        .await;
8423
8424        let developer = request
8425            .instructions
8426            .developer
8427            .as_deref()
8428            .expect("ultra mode developer instructions");
8429        assert!(developer.contains("Proactive multi-agent delegation is active"));
8430        assert!(developer.contains("spawn_agent"));
8431    }
8432
8433    #[tokio::test]
8434    async fn ultra_mode_overrides_explicit_request_only_on_sol() {
8435        // Medium Sol effort is normally explicit-request-only; ultra mode flips
8436        // it to proactive without requiring Ultra reasoning effort.
8437        let request = captured_profile_request(RuntimeConfig {
8438            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8439            default_model: "gpt-5.6-sol".to_string(),
8440            reasoning: Some(roder_api::catalog::REASONING_MEDIUM.to_string()),
8441            ultra_mode: true,
8442            ..RuntimeConfig::default()
8443        })
8444        .await;
8445
8446        let developer = request
8447            .instructions
8448            .developer
8449            .as_deref()
8450            .expect("ultra mode sol developer instructions");
8451        assert!(developer.contains("Proactive multi-agent delegation is active"));
8452        assert!(!developer.contains("Do not spawn sub-agents unless"));
8453    }
8454
8455    #[tokio::test]
8456    async fn turn_developer_context_reaches_inference_and_does_not_persist() {
8457        let captured = Arc::new(StdMutex::new(None));
8458        let mut builder = ExtensionRegistryBuilder::new();
8459        builder.inference_engine(Arc::new(CapturingEngine {
8460            request: captured.clone(),
8461        }));
8462        let runtime = Arc::new(
8463            Runtime::new(
8464                builder.build().unwrap(),
8465                RuntimeConfig {
8466                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8467                    default_model: "gpt-5.5".to_string(),
8468                    ..RuntimeConfig::default()
8469                },
8470            )
8471            .unwrap(),
8472        );
8473
8474        async fn run_turn(runtime: &Arc<Runtime>, developer_context: Option<String>) {
8475            let mut rx = runtime.subscribe_events();
8476            let turn_id = runtime
8477                .start_turn(StartTurnRequest {
8478                    thread_id: "thread-turn-context".to_string(),
8479                    message: "hello".to_string(),
8480                    images: Vec::new(),
8481                    provider_override: None,
8482                    model_override: None,
8483                    reasoning_override: None,
8484                    workspace: test_workspace(),
8485                    instructions: InstructionBundle::default(),
8486                    developer_context,
8487                    task_ledger_required: false,
8488                    service_tier_override: None,
8489                })
8490                .await
8491                .unwrap();
8492            tokio::time::timeout(std::time::Duration::from_secs(5), async {
8493                loop {
8494                    let envelope = rx.recv().await.unwrap();
8495                    if envelope.turn_id.as_deref() != Some(&turn_id) {
8496                        continue;
8497                    }
8498                    match envelope.event {
8499                        RoderEvent::TurnCompleted(_) => break,
8500                        RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8501                        _ => {}
8502                    }
8503                }
8504            })
8505            .await
8506            .unwrap();
8507        }
8508
8509        run_turn(
8510            &runtime,
8511            Some("Connected accounts: example-service.".to_string()),
8512        )
8513        .await;
8514        let request = captured.lock().unwrap().clone().unwrap();
8515        assert_eq!(
8516            request.instructions.developer_context.as_deref(),
8517            Some("Connected accounts: example-service.")
8518        );
8519
8520        // The context is per-turn only: the next turn on the same thread
8521        // without a developerContext must not see the previous one.
8522        run_turn(&runtime, None).await;
8523        let request = captured.lock().unwrap().clone().unwrap();
8524        assert_eq!(request.instructions.developer_context, None);
8525    }
8526
8527    #[tokio::test]
8528    async fn tool_search_overrides_route_to_next_inference_request() {
8529        let request = captured_profile_request(RuntimeConfig {
8530            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8531            default_model: "gpt-5.4".to_string(),
8532            tool_search: ToolSearchConfig {
8533                mode: roder_api::inference::ToolSearchMode::Auto,
8534                max_catalog_items: Some(100),
8535                ..ToolSearchConfig::default()
8536            },
8537            provider_tool_search: std::collections::HashMap::from([(
8538                roder_api::catalog::PROVIDER_MOCK.to_string(),
8539                roder_api::inference::ToolSearchConfigOverlay {
8540                    include_skills: Some(false),
8541                    provider_variant: Some(roder_api::inference::ToolSearchProviderVariant::Regex),
8542                    ..Default::default()
8543                },
8544            )]),
8545            model_tool_search: std::collections::HashMap::from([(
8546                "gpt-5.4".to_string(),
8547                roder_api::inference::ToolSearchConfigOverlay {
8548                    mode: Some(roder_api::inference::ToolSearchMode::ProviderNative),
8549                    max_catalog_items: Some(25),
8550                    provider_variant: Some(roder_api::inference::ToolSearchProviderVariant::Bm25),
8551                    ..Default::default()
8552                },
8553            )]),
8554            ..RuntimeConfig::default()
8555        })
8556        .await;
8557
8558        assert_eq!(
8559            request.runtime.tool_search.mode,
8560            roder_api::inference::ToolSearchMode::ProviderNative
8561        );
8562        assert_eq!(request.runtime.tool_search.max_catalog_items, Some(25));
8563        assert_eq!(
8564            request.runtime.tool_search.provider_variant,
8565            roder_api::inference::ToolSearchProviderVariant::Bm25
8566        );
8567    }
8568
8569    #[tokio::test]
8570    async fn context_entrypoint_hints_use_turn_workspace() {
8571        let process_workspace = runtime_test_workspace("entrypoint-process");
8572        let thread_workspace = runtime_test_workspace("entrypoint-thread");
8573        std::fs::create_dir_all(process_workspace.join("src")).unwrap();
8574        std::fs::create_dir_all(thread_workspace.join("src")).unwrap();
8575        std::fs::write(
8576            process_workspace.join("src/sidebar-thread-groups.ts"),
8577            "export const desktopLeak = true;\n",
8578        )
8579        .unwrap();
8580        std::fs::write(
8581            thread_workspace.join("src/voice-plan-feedback.ts"),
8582            "export const voicePlanFeedback = true;\n",
8583        )
8584        .unwrap();
8585
8586        let captured = Arc::new(StdMutex::new(None));
8587        let mut builder = ExtensionRegistryBuilder::new();
8588        builder.inference_engine(Arc::new(CapturingEngine {
8589            request: captured.clone(),
8590        }));
8591        builder.context_planner(Arc::new(roder_context::EntrypointContextPlanner::new(
8592            process_workspace.clone(),
8593        )));
8594        let runtime = Arc::new(
8595            Runtime::new(
8596                builder.build().unwrap(),
8597                RuntimeConfig {
8598                    workspace: Some(process_workspace.display().to_string()),
8599                    ..RuntimeConfig::default()
8600                },
8601            )
8602            .unwrap(),
8603        );
8604        let mut rx = runtime.subscribe_events();
8605
8606        let turn_id = runtime
8607            .start_turn(StartTurnRequest {
8608                thread_id: "thread-workspace-entrypoint".to_string(),
8609                message: "investigate voice plan feedback".to_string(),
8610                images: Vec::new(),
8611                provider_override: None,
8612                model_override: None,
8613                reasoning_override: None,
8614                workspace: thread_workspace.display().to_string(),
8615                instructions: crate::instructions::default_instructions(),
8616                developer_context: None,
8617                task_ledger_required: false,
8618                service_tier_override: None,
8619            })
8620            .await
8621            .unwrap();
8622
8623        tokio::time::timeout(std::time::Duration::from_secs(5), async {
8624            loop {
8625                let envelope = rx.recv().await.unwrap();
8626                if envelope.turn_id.as_deref() != Some(&turn_id) {
8627                    continue;
8628                }
8629                match envelope.event {
8630                    RoderEvent::TurnCompleted(_) => break,
8631                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8632                    _ => {}
8633                }
8634            }
8635        })
8636        .await
8637        .unwrap();
8638
8639        let request = captured.lock().unwrap().clone().expect("captured request");
8640        let transcript_text = request
8641            .transcript
8642            .iter()
8643            .map(|item| match item {
8644                TranscriptItem::UserMessage(message) => message.text.as_str(),
8645                _ => "",
8646            })
8647            .collect::<Vec<_>>()
8648            .join("\n");
8649        assert!(transcript_text.contains("src/voice-plan-feedback.ts"));
8650        assert!(!transcript_text.contains("src/sidebar-thread-groups.ts"));
8651
8652        let _ = std::fs::remove_dir_all(process_workspace);
8653        let _ = std::fs::remove_dir_all(thread_workspace);
8654    }
8655
8656    fn runtime_test_workspace(name: &str) -> std::path::PathBuf {
8657        let path =
8658            std::env::temp_dir().join(format!("roder-runtime-{name}-{}", uuid::Uuid::new_v4()));
8659        let _ = std::fs::remove_dir_all(&path);
8660        std::fs::create_dir_all(&path).unwrap();
8661        path
8662    }
8663
8664    #[tokio::test]
8665    async fn model_profile_user_model_knobs_override_profile_defaults() {
8666        let request = captured_profile_request(RuntimeConfig {
8667            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8668            default_model: "gpt-5.5".to_string(),
8669            reasoning: Some(REASONING_HIGH.to_string()),
8670            auto_compact_token_limit: Some(999),
8671            model_edit_tools: std::collections::HashMap::from([(
8672                "gpt-5.5".to_string(),
8673                EDIT_TOOL_PATCH.to_string(),
8674            )]),
8675            model_parallel_tool_calls: std::collections::HashMap::from([(
8676                "gpt-5.5".to_string(),
8677                true,
8678            )]),
8679            model_profiles: std::collections::HashMap::from([(
8680                "gpt-5.5".to_string(),
8681                test_model_profile("gpt-5.5"),
8682            )]),
8683            ..RuntimeConfig::default()
8684        })
8685        .await;
8686
8687        let tool_names = request
8688            .tools
8689            .iter()
8690            .map(|tool| tool.name.as_str())
8691            .collect::<Vec<_>>();
8692        assert!(tool_names.contains(&"apply_patch"));
8693        assert!(!tool_names.contains(&"edit"));
8694        assert_eq!(request.reasoning.level.as_deref(), Some(REASONING_HIGH));
8695        assert_eq!(request.runtime.parallel_tool_calls, Some(true));
8696        assert_eq!(request.runtime.auto_compact_token_limit, Some(999));
8697    }
8698
8699    #[tokio::test]
8700    async fn runtime_tool_allowlist_filters_advertised_tools() {
8701        let request = captured_profile_request(RuntimeConfig {
8702            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8703            default_model: "gpt-5.5".to_string(),
8704            tool_allowlist: vec!["edit".to_string()],
8705            model_profiles: std::collections::HashMap::from([(
8706                "gpt-5.5".to_string(),
8707                test_model_profile("gpt-5.5"),
8708            )]),
8709            ..RuntimeConfig::default()
8710        })
8711        .await;
8712
8713        let tool_names = request
8714            .tools
8715            .iter()
8716            .map(|tool| tool.name.as_str())
8717            .collect::<Vec<_>>();
8718        assert_eq!(tool_names, vec!["edit"]);
8719    }
8720
8721    /// Builds a store-backed runtime turn for a thread created with the given per-thread
8722    /// overrides and returns the captured inference request.
8723    async fn captured_thread_override_request(
8724        runtime: &Arc<Runtime>,
8725        requests: &Arc<StdMutex<Vec<AgentInferenceRequest>>>,
8726        tool_allowlist: Vec<String>,
8727        developer_instructions: Option<String>,
8728        external_tools: Vec<ToolSpec>,
8729    ) -> AgentInferenceRequest {
8730        let thread_id = runtime
8731            .create_thread_with(CreateThreadRequest {
8732                title: Some("Thread overrides".to_string()),
8733                workspace: test_workspace(),
8734                workspace_id: None,
8735                root_id: None,
8736                provider: None,
8737                model: None,
8738                selection_mode: None,
8739                tool_allowlist,
8740                developer_instructions,
8741                external_tools,
8742                runner: None,
8743            })
8744            .await
8745            .unwrap()
8746            .thread_id;
8747        let mut rx = runtime.subscribe_events();
8748        let turn_id = runtime
8749            .start_turn(StartTurnRequest {
8750                thread_id,
8751                message: "hello".to_string(),
8752                images: Vec::new(),
8753                provider_override: None,
8754                model_override: None,
8755                reasoning_override: None,
8756                workspace: test_workspace(),
8757                instructions: crate::default_instructions(),
8758                developer_context: None,
8759                task_ledger_required: false,
8760                service_tier_override: None,
8761            })
8762            .await
8763            .unwrap();
8764        tokio::time::timeout(std::time::Duration::from_secs(5), async {
8765            loop {
8766                let envelope = rx.recv().await.unwrap();
8767                if envelope.turn_id.as_deref() != Some(&turn_id) {
8768                    continue;
8769                }
8770                match envelope.event {
8771                    RoderEvent::TurnCompleted(_) => break,
8772                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8773                    _ => {}
8774                }
8775            }
8776        })
8777        .await
8778        .unwrap();
8779        requests.lock().unwrap().pop().expect("captured request")
8780    }
8781
8782    #[tokio::test]
8783    async fn thread_tool_allowlist_filters_only_that_thread() {
8784        let requests = Arc::new(StdMutex::new(Vec::new()));
8785        let thread_root =
8786            std::env::temp_dir().join(format!("roder-thread-allowlist-{}", uuid::Uuid::new_v4()));
8787        let mut builder = ExtensionRegistryBuilder::new();
8788        builder.inference_engine(Arc::new(SwitchCaptureEngine {
8789            requests: requests.clone(),
8790        }));
8791        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8792            base_path: thread_root.clone(),
8793        }));
8794        builder.tool_contributor(Arc::new(ProfileToolContributor));
8795        let runtime = Arc::new(
8796            Runtime::new(
8797                builder.build().unwrap(),
8798                RuntimeConfig {
8799                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8800                    default_model: "gpt-5.5".to_string(),
8801                    model_profiles: std::collections::HashMap::from([(
8802                        "gpt-5.5".to_string(),
8803                        test_model_profile("gpt-5.5"),
8804                    )]),
8805                    ..RuntimeConfig::default()
8806                },
8807            )
8808            .unwrap(),
8809        );
8810
8811        let allowlisted = captured_thread_override_request(
8812            &runtime,
8813            &requests,
8814            vec!["edit".to_string()],
8815            None,
8816            Vec::new(),
8817        )
8818        .await;
8819        let unrestricted =
8820            captured_thread_override_request(&runtime, &requests, Vec::new(), None, Vec::new())
8821                .await;
8822
8823        let allowlisted_names = allowlisted
8824            .tools
8825            .iter()
8826            .map(|tool| tool.name.as_str())
8827            .collect::<Vec<_>>();
8828        assert_eq!(allowlisted_names, vec!["edit"]);
8829        let unrestricted_names = unrestricted
8830            .tools
8831            .iter()
8832            .map(|tool| tool.name.as_str())
8833            .collect::<Vec<_>>();
8834        assert!(unrestricted_names.contains(&"edit"));
8835        assert!(unrestricted_names.len() > 1);
8836
8837        let _ = std::fs::remove_dir_all(thread_root);
8838    }
8839
8840    /// Builds a store-backed runtime whose `RuntimeConfig.tool_allowlist` is `["edit"]`.
8841    fn runtime_with_edit_allowlist(
8842        requests: &Arc<StdMutex<Vec<AgentInferenceRequest>>>,
8843        thread_root: &std::path::Path,
8844    ) -> Arc<Runtime> {
8845        let mut builder = ExtensionRegistryBuilder::new();
8846        builder.inference_engine(Arc::new(SwitchCaptureEngine {
8847            requests: requests.clone(),
8848        }));
8849        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8850            base_path: thread_root.to_path_buf(),
8851        }));
8852        builder.tool_contributor(Arc::new(ProfileToolContributor));
8853        Arc::new(
8854            Runtime::new(
8855                builder.build().unwrap(),
8856                RuntimeConfig {
8857                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8858                    default_model: "gpt-5.5".to_string(),
8859                    tool_allowlist: vec!["edit".to_string()],
8860                    model_profiles: std::collections::HashMap::from([(
8861                        "gpt-5.5".to_string(),
8862                        test_model_profile("gpt-5.5"),
8863                    )]),
8864                    ..RuntimeConfig::default()
8865                },
8866            )
8867            .unwrap(),
8868        )
8869    }
8870
8871    #[tokio::test]
8872    async fn runtime_and_thread_allowlists_intersect() {
8873        let requests = Arc::new(StdMutex::new(Vec::new()));
8874        let thread_root = std::env::temp_dir().join(format!(
8875            "roder-allowlist-intersect-{}",
8876            uuid::Uuid::new_v4()
8877        ));
8878        let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8879
8880        let request = captured_thread_override_request(
8881            &runtime,
8882            &requests,
8883            vec!["edit".to_string(), "write_file".to_string()],
8884            None,
8885            Vec::new(),
8886        )
8887        .await;
8888
8889        // The thread allowlist must not re-enable tools the runtime allowlist bans.
8890        let names = request
8891            .tools
8892            .iter()
8893            .map(|tool| tool.name.as_str())
8894            .collect::<Vec<_>>();
8895        assert_eq!(names, vec!["edit"]);
8896
8897        let _ = std::fs::remove_dir_all(thread_root);
8898    }
8899
8900    #[tokio::test]
8901    async fn route_tool_call_denies_tools_outside_allowlists() {
8902        let requests = Arc::new(StdMutex::new(Vec::new()));
8903        let thread_root =
8904            std::env::temp_dir().join(format!("roder-allowlist-dispatch-{}", uuid::Uuid::new_v4()));
8905        let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8906        let thread_id = runtime
8907            .create_thread_with(CreateThreadRequest {
8908                title: Some("Dispatch allowlist".to_string()),
8909                workspace: test_workspace(),
8910                workspace_id: None,
8911                root_id: None,
8912                provider: None,
8913                model: None,
8914                selection_mode: None,
8915                tool_allowlist: vec!["edit".to_string(), "write_file".to_string()],
8916                developer_instructions: None,
8917                external_tools: Vec::new(),
8918                runner: None,
8919            })
8920            .await
8921            .unwrap()
8922            .thread_id;
8923
8924        // write_file is registered and on the thread allowlist but banned by the runtime allowlist.
8925        let result = runtime
8926            .route_tool_call(
8927                &thread_id,
8928                &"turn-allowlist-dispatch".to_string(),
8929                roder_api::inference::ToolCallCompleted {
8930                    id: "call-1".to_string(),
8931                    name: "write_file".to_string(),
8932                    arguments: r#"{"path":"a.txt","content":"hi"}"#.to_string(),
8933                },
8934                None,
8935                None,
8936            )
8937            .await
8938            .unwrap();
8939
8940        assert!(result.is_error);
8941        assert!(
8942            result
8943                .result
8944                .contains("not permitted by the tool allowlist"),
8945            "unexpected result: {}",
8946            result.result
8947        );
8948
8949        let _ = std::fs::remove_dir_all(thread_root);
8950    }
8951
8952    #[tokio::test]
8953    async fn route_tool_calls_denies_agent_swarm_mixed_with_other_tools() {
8954        let requests = Arc::new(StdMutex::new(Vec::new()));
8955        let thread_root =
8956            std::env::temp_dir().join(format!("roder-swarm-exclusive-{}", uuid::Uuid::new_v4()));
8957        let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8958
8959        // A response that mixes agent_swarm with another tool is denied wholesale:
8960        // both calls get an error result with retry guidance, and no tool runs.
8961        let results = runtime
8962            .route_tool_calls(
8963                &"thread-swarm".to_string(),
8964                &"turn-swarm".to_string(),
8965                vec![
8966                    roder_api::inference::ToolCallCompleted {
8967                        id: "swarm-1".to_string(),
8968                        name: "agent_swarm".to_string(),
8969                        arguments: "{}".to_string(),
8970                    },
8971                    roder_api::inference::ToolCallCompleted {
8972                        id: "read-1".to_string(),
8973                        name: "read_file".to_string(),
8974                        arguments: "{}".to_string(),
8975                    },
8976                ],
8977                true,
8978                None,
8979                None,
8980            )
8981            .await
8982            .unwrap();
8983
8984        assert_eq!(results.len(), 2, "every tool_call_id must get a response");
8985        assert_eq!(results[0].id, "swarm-1");
8986        assert_eq!(results[1].id, "read-1");
8987        for result in &results {
8988            assert!(result.is_error);
8989            assert!(
8990                result.result.contains("only tool call"),
8991                "unexpected result: {}",
8992                result.result
8993            );
8994        }
8995
8996        let _ = std::fs::remove_dir_all(thread_root);
8997    }
8998
8999    #[tokio::test]
9000    async fn route_tool_calls_emits_agent_swarm_started_event() {
9001        let requests = Arc::new(StdMutex::new(Vec::new()));
9002        let thread_root =
9003            std::env::temp_dir().join(format!("roder-swarm-started-{}", uuid::Uuid::new_v4()));
9004        let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
9005        let mut events = runtime.subscribe_events();
9006
9007        // A single agent_swarm call (the valid shape) emits AgentSwarmStarted on
9008        // the event bus before dispatch, with the child count parsed from args.
9009        let _ = runtime
9010            .route_tool_calls(
9011                &"thread-swarm".to_string(),
9012                &"turn-swarm".to_string(),
9013                vec![roder_api::inference::ToolCallCompleted {
9014                    id: "swarm-1".to_string(),
9015                    name: "agent_swarm".to_string(),
9016                    arguments: r#"{"description":"x","prompt_template":"Read {{item}}","items":["a.rs","b.rs"]}"#
9017                        .to_string(),
9018                }],
9019                true,
9020                None,
9021                None,
9022            )
9023            .await
9024            .unwrap();
9025
9026        let mut started_child_count = None;
9027        for _ in 0..16 {
9028            let envelope = tokio::time::timeout(std::time::Duration::from_secs(2), events.recv())
9029                .await
9030                .unwrap()
9031                .unwrap();
9032            if let RoderEvent::AgentSwarmStarted(event) = envelope.event {
9033                started_child_count = Some(event.child_count);
9034                assert_eq!(event.tool_id, "swarm-1");
9035                break;
9036            }
9037        }
9038        assert_eq!(started_child_count, Some(2));
9039
9040        let _ = std::fs::remove_dir_all(thread_root);
9041    }
9042
9043    #[test]
9044    fn agent_swarm_child_count_sums_items_and_resumes() {
9045        assert_eq!(
9046            agent_swarm_child_count(r#"{"description":"x","items":["a","b","c"]}"#),
9047            3
9048        );
9049        assert_eq!(
9050            agent_swarm_child_count(
9051                r#"{"description":"x","items":["a"],"resume_agent_ids":{"id1":"continue"}}"#
9052            ),
9053            2
9054        );
9055        // Malformed input is lenient.
9056        assert_eq!(agent_swarm_child_count("not json"), 0);
9057    }
9058
9059    #[test]
9060    fn parse_swarm_counts_reads_summary_with_omitted_buckets() {
9061        let text = "<agent_swarm_result>\n<summary>completed: 2, failed: 1</summary>\n</agent_swarm_result>";
9062        assert_eq!(parse_swarm_counts(text), Some((2, 1, 0)));
9063        let text = "<agent_swarm_result>\n<summary>completed: 0</summary>\n</agent_swarm_result>";
9064        assert_eq!(parse_swarm_counts(text), Some((0, 0, 0)));
9065        // Not a swarm result.
9066        assert_eq!(parse_swarm_counts("just text"), None);
9067    }
9068
9069    /// Signals when inference starts, then waits for `proceed` and fails the stream.
9070    struct SignalledFailureEngine {
9071        started: tokio::sync::mpsc::UnboundedSender<()>,
9072        proceed: Arc<tokio::sync::Notify>,
9073    }
9074
9075    #[async_trait::async_trait]
9076    impl InferenceEngine for SignalledFailureEngine {
9077        fn id(&self) -> String {
9078            roder_api::catalog::PROVIDER_MOCK.to_string()
9079        }
9080
9081        fn capabilities(&self) -> InferenceCapabilities {
9082            InferenceCapabilities::coding_agent_default()
9083        }
9084
9085        async fn list_models(
9086            &self,
9087            _ctx: InferenceProviderContext<'_>,
9088        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
9089            Ok(roder_api::catalog::models_for_provider(
9090                roder_api::catalog::PROVIDER_MOCK,
9091                true,
9092            ))
9093        }
9094
9095        async fn stream_turn(
9096            &self,
9097            _ctx: InferenceTurnContext<'_>,
9098            _request: AgentInferenceRequest,
9099        ) -> anyhow::Result<InferenceEventStream> {
9100            let _ = self.started.send(());
9101            self.proceed.notified().await;
9102            anyhow::bail!("engine failed mid-turn")
9103        }
9104    }
9105
9106    #[tokio::test]
9107    async fn failed_turn_sweeps_pending_external_tool_calls() {
9108        let (started_tx, mut started_rx) = tokio::sync::mpsc::unbounded_channel();
9109        let proceed = Arc::new(tokio::sync::Notify::new());
9110        let mut builder = ExtensionRegistryBuilder::new();
9111        builder.inference_engine(Arc::new(SignalledFailureEngine {
9112            started: started_tx,
9113            proceed: proceed.clone(),
9114        }));
9115        let runtime =
9116            Arc::new(Runtime::new(builder.build().unwrap(), RuntimeConfig::default()).unwrap());
9117        let mut rx = runtime.subscribe_events();
9118        let turn_id = runtime
9119            .start_turn(StartTurnRequest {
9120                thread_id: "thread-sweep".to_string(),
9121                message: "go".to_string(),
9122                images: Vec::new(),
9123                provider_override: None,
9124                model_override: None,
9125                reasoning_override: None,
9126                workspace: test_workspace(),
9127                instructions: InstructionBundle {
9128                    system: None,
9129                    developer: None,
9130                    developer_context: None,
9131                },
9132                developer_context: None,
9133                task_ledger_required: false,
9134                service_tier_override: None,
9135            })
9136            .await
9137            .unwrap();
9138        tokio::time::timeout(std::time::Duration::from_secs(5), started_rx.recv())
9139            .await
9140            .unwrap()
9141            .unwrap();
9142
9143        let (tx, _pending_rx) = oneshot::channel();
9144        runtime.pending_external_tool_calls.lock().await.insert(
9145            "exttool-sweep-test".to_string(),
9146            PendingExternalToolCall {
9147                thread_id: "thread-sweep".to_string(),
9148                turn_id: turn_id.clone(),
9149                tool_id: "call-1".to_string(),
9150                tool_name: "acme_lookup".to_string(),
9151                tx,
9152            },
9153        );
9154        proceed.notify_one();
9155
9156        let outcome = tokio::time::timeout(std::time::Duration::from_secs(5), async {
9157            loop {
9158                let envelope = rx.recv().await.unwrap();
9159                if let RoderEvent::ExternalToolCallResolved(event) = envelope.event
9160                    && event.request_id == "exttool-sweep-test"
9161                {
9162                    break event.outcome;
9163                }
9164            }
9165        })
9166        .await
9167        .expect("turn failure must resolve pending external tool calls");
9168        assert_eq!(outcome, ExternalToolCallOutcome::Cancelled);
9169        assert!(runtime.pending_external_tool_calls.lock().await.is_empty());
9170    }
9171
9172    #[tokio::test]
9173    async fn thread_developer_instructions_layer_under_harness_prompt() {
9174        let requests = Arc::new(StdMutex::new(Vec::new()));
9175        let thread_root = std::env::temp_dir().join(format!(
9176            "roder-thread-instructions-{}",
9177            uuid::Uuid::new_v4()
9178        ));
9179        let mut builder = ExtensionRegistryBuilder::new();
9180        builder.inference_engine(Arc::new(SwitchCaptureEngine {
9181            requests: requests.clone(),
9182        }));
9183        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
9184            base_path: thread_root.clone(),
9185        }));
9186        builder.tool_contributor(Arc::new(ProfileToolContributor));
9187        let runtime =
9188            Arc::new(Runtime::new(builder.build().unwrap(), RuntimeConfig::default()).unwrap());
9189
9190        let request = captured_thread_override_request(
9191            &runtime,
9192            &requests,
9193            Vec::new(),
9194            Some("You are embedded in a host app.".to_string()),
9195            Vec::new(),
9196        )
9197        .await;
9198
9199        let system = request.instructions.system.expect("system instructions");
9200        assert!(system.starts_with("You are Roder"));
9201        let developer = request
9202            .instructions
9203            .developer
9204            .expect("developer instructions");
9205        assert!(developer.starts_with("You are embedded in a host app."));
9206
9207        let plain =
9208            captured_thread_override_request(&runtime, &requests, Vec::new(), None, Vec::new())
9209                .await;
9210        assert_eq!(plain.instructions.developer, None);
9211
9212        let _ = std::fs::remove_dir_all(thread_root);
9213    }
9214
9215    #[tokio::test]
9216    async fn thread_external_tools_are_advertised_and_shadow_builtins() {
9217        let requests = Arc::new(StdMutex::new(Vec::new()));
9218        let thread_root = std::env::temp_dir().join(format!(
9219            "roder-thread-external-tools-{}",
9220            uuid::Uuid::new_v4()
9221        ));
9222        let mut builder = ExtensionRegistryBuilder::new();
9223        builder.inference_engine(Arc::new(SwitchCaptureEngine {
9224            requests: requests.clone(),
9225        }));
9226        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
9227            base_path: thread_root.clone(),
9228        }));
9229        builder.tool_contributor(Arc::new(ProfileToolContributor));
9230        let runtime =
9231            Arc::new(Runtime::new(builder.build().unwrap(), RuntimeConfig::default()).unwrap());
9232
9233        let external_tools = vec![
9234            ToolSpec {
9235                name: "acme_lookup".to_string(),
9236                description: "Look up Acme workspace state.".to_string(),
9237                parameters: serde_json::json!({
9238                    "type": "object",
9239                    "properties": { "query": { "type": "string" } },
9240                    "required": ["query"]
9241                }),
9242            },
9243            ToolSpec {
9244                name: "edit".to_string(),
9245                description: "Host-managed edit.".to_string(),
9246                parameters: serde_json::json!({ "type": "object" }),
9247            },
9248        ];
9249        let request =
9250            captured_thread_override_request(&runtime, &requests, Vec::new(), None, external_tools)
9251                .await;
9252
9253        let acme = request
9254            .tools
9255            .iter()
9256            .find(|tool| tool.name == "acme_lookup")
9257            .expect("external tool advertised");
9258        assert_eq!(acme.description, "Look up Acme workspace state.");
9259        assert_eq!(acme.parameters["required"][0], "query");
9260        let edits = request
9261            .tools
9262            .iter()
9263            .filter(|tool| tool.name == "edit")
9264            .collect::<Vec<_>>();
9265        assert_eq!(edits.len(), 1, "external edit shadows the builtin");
9266        assert_eq!(edits[0].description, "Host-managed edit.");
9267
9268        let plain =
9269            captured_thread_override_request(&runtime, &requests, Vec::new(), None, Vec::new())
9270                .await;
9271        assert!(plain.tools.iter().all(|tool| tool.name != "acme_lookup"));
9272        let plain_edit = plain
9273            .tools
9274            .iter()
9275            .find(|tool| tool.name == "edit")
9276            .expect("builtin edit advertised on plain thread");
9277        assert_eq!(plain_edit.description, "edit test tool");
9278
9279        let _ = std::fs::remove_dir_all(thread_root);
9280    }
9281
9282    #[tokio::test]
9283    async fn model_switch_injects_summary_and_records_profile_segments() {
9284        let requests = Arc::new(StdMutex::new(Vec::new()));
9285        let thread_root = std::env::temp_dir().join(format!(
9286            "roder-model-switch-thread-{}",
9287            uuid::Uuid::new_v4()
9288        ));
9289        let mut builder = ExtensionRegistryBuilder::new();
9290        builder.inference_engine(Arc::new(SwitchCaptureEngine {
9291            requests: requests.clone(),
9292        }));
9293        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
9294            base_path: thread_root.clone(),
9295        }));
9296        builder.tool_contributor(Arc::new(ProfileToolContributor));
9297        let mut claude_profile = test_model_profile("claude-haiku-4-5-20251001");
9298        claude_profile.provider_family = ProviderFamily::Anthropic;
9299        claude_profile.edit_tool = Some(EDIT_TOOL_EDIT.to_string());
9300        let runtime = Arc::new(
9301            Runtime::new(
9302                builder.build().unwrap(),
9303                RuntimeConfig {
9304                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9305                    default_model: "gpt-5.5".to_string(),
9306                    model_profiles: std::collections::HashMap::from([
9307                        ("gpt-5.5".to_string(), test_model_profile("gpt-5.5")),
9308                        ("claude-haiku-4-5-20251001".to_string(), claude_profile),
9309                    ]),
9310                    ..RuntimeConfig::default()
9311                },
9312            )
9313            .unwrap(),
9314        );
9315        let thread_id = runtime
9316            .create_thread_with(CreateThreadRequest {
9317                title: Some("Model switch".to_string()),
9318                workspace: test_workspace(),
9319                workspace_id: None,
9320                root_id: None,
9321                provider: None,
9322                model: None,
9323                selection_mode: None,
9324                tool_allowlist: Vec::new(),
9325                developer_instructions: None,
9326                external_tools: Vec::new(),
9327                runner: None,
9328            })
9329            .await
9330            .unwrap()
9331            .thread_id;
9332        let mut rx = runtime.subscribe_events();
9333        for (message, model_override) in [
9334            ("first turn", None),
9335            ("second turn", Some("claude-haiku-4-5-20251001".to_string())),
9336        ] {
9337            let turn_id = runtime
9338                .start_turn(StartTurnRequest {
9339                    thread_id: thread_id.clone(),
9340                    message: message.to_string(),
9341                    images: Vec::new(),
9342                    provider_override: None,
9343                    model_override,
9344                    reasoning_override: None,
9345                    workspace: test_workspace(),
9346                    instructions: InstructionBundle::default(),
9347                    developer_context: None,
9348                    task_ledger_required: false,
9349                    service_tier_override: None,
9350                })
9351                .await
9352                .unwrap();
9353            tokio::time::timeout(std::time::Duration::from_secs(5), async {
9354                loop {
9355                    let envelope = rx.recv().await.unwrap();
9356                    if envelope.turn_id.as_deref() != Some(&turn_id) {
9357                        continue;
9358                    }
9359                    match envelope.event {
9360                        RoderEvent::TurnCompleted(_) => break,
9361                        RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9362                        _ => {}
9363                    }
9364                }
9365            })
9366            .await
9367            .unwrap();
9368        }
9369
9370        let captured = requests.lock().unwrap().clone();
9371        assert_eq!(captured.len(), 2);
9372        assert!(captured[1].transcript.iter().any(|item| {
9373            matches!(
9374                item,
9375                TranscriptItem::UserMessage(message)
9376                    if message.text.starts_with(MODEL_SWITCH_SUMMARY_PREFIX)
9377                        && message.text.contains("previous profile mock/gpt-5.5")
9378                        && message.text.contains("Current profile mock/claude-haiku-4-5-20251001")
9379                        && message.text.contains("Available tools now:")
9380            )
9381        }));
9382
9383        let snapshot = runtime
9384            .thread_store
9385            .as_ref()
9386            .unwrap()
9387            .load_thread(&thread_id)
9388            .await
9389            .unwrap()
9390            .unwrap();
9391        let trace_segments = snapshot
9392            .turns
9393            .iter()
9394            .flat_map(|turn| &turn.items)
9395            .filter(|item| {
9396                matches!(
9397                    item,
9398                    TranscriptItem::ProviderMetadata(value)
9399                        if value.get("kind").and_then(serde_json::Value::as_str)
9400                            == Some(MODEL_PROFILE_TRACE_KIND)
9401                            && value.get("segment").and_then(serde_json::Value::as_str)
9402                                == Some("assistant")
9403                )
9404            })
9405            .count();
9406        assert!(trace_segments >= 2);
9407        let _ = std::fs::remove_dir_all(thread_root);
9408    }
9409
9410    struct CountingTaskTool {
9411        calls: Arc<StdMutex<u32>>,
9412    }
9413
9414    #[async_trait::async_trait]
9415    impl ToolExecutor for CountingTaskTool {
9416        fn spec(&self) -> ToolSpec {
9417            ToolSpec {
9418                name: "task".to_string(),
9419                description: "Dispatch a test subagent.".to_string(),
9420                parameters: serde_json::json!({
9421                    "type": "object",
9422                    "properties": {
9423                        "description": { "type": "string" },
9424                        "prompt": { "type": "string" },
9425                        "parent_deadline_seconds": { "type": "integer" }
9426                    },
9427                    "required": ["description", "prompt"],
9428                    "additionalProperties": false
9429                }),
9430            }
9431        }
9432
9433        async fn execute(
9434            &self,
9435            _ctx: ToolExecutionContext,
9436            call: ToolCall,
9437        ) -> anyhow::Result<ToolResult> {
9438            *self.calls.lock().unwrap() += 1;
9439            Ok(ToolResult {
9440                id: call.id,
9441                name: call.name,
9442                text: "started child".to_string(),
9443                data: serde_json::json!({}),
9444                is_error: false,
9445            })
9446        }
9447    }
9448
9449    #[tokio::test]
9450    async fn runtime_profile_reaches_inference_request_and_turn_metadata() {
9451        let captured = Arc::new(StdMutex::new(None));
9452        let mut builder = ExtensionRegistryBuilder::new();
9453        builder.inference_engine(Arc::new(CapturingEngine {
9454            request: captured.clone(),
9455        }));
9456        let runtime = Arc::new(
9457            Runtime::new(
9458                builder.build().unwrap(),
9459                RuntimeConfig {
9460                    runtime_profile: RuntimeProfile::NonInteractive,
9461                    ..RuntimeConfig::default()
9462                },
9463            )
9464            .unwrap(),
9465        );
9466        let mut rx = runtime.subscribe_events();
9467        let turn_id = runtime
9468            .start_turn(StartTurnRequest {
9469                thread_id: "thread-profile".to_string(),
9470                message: "work unattended".to_string(),
9471                images: Vec::new(),
9472                provider_override: None,
9473                model_override: None,
9474                reasoning_override: None,
9475                workspace: test_workspace(),
9476                instructions: InstructionBundle {
9477                    system: None,
9478                    developer: Some("base developer".to_string()),
9479                    developer_context: None,
9480                },
9481                developer_context: None,
9482                task_ledger_required: false,
9483                service_tier_override: None,
9484            })
9485            .await
9486            .unwrap();
9487
9488        let mut observed_profile = None;
9489        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9490            loop {
9491                let envelope = rx.recv().await.unwrap();
9492                if envelope.turn_id.as_deref() != Some(&turn_id) {
9493                    continue;
9494                }
9495                match envelope.event {
9496                    RoderEvent::TurnStarted(event) => {
9497                        observed_profile = Some(event.runtime_profile);
9498                    }
9499                    RoderEvent::TurnCompleted(_) => break,
9500                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9501                    _ => {}
9502                }
9503            }
9504        })
9505        .await
9506        .unwrap();
9507
9508        assert_eq!(observed_profile, Some(RuntimeProfile::NonInteractive));
9509        let request = captured.lock().unwrap().clone().unwrap();
9510        assert_eq!(request.runtime.profile, RuntimeProfile::NonInteractive);
9511        let developer = request.instructions.developer.unwrap();
9512        assert!(developer.contains("base developer"));
9513        assert!(developer.contains("non-interactive profile"));
9514    }
9515
9516    #[tokio::test]
9517    async fn global_policy_mode_changes_do_not_create_runtime_thread_directory() {
9518        let workspace = runtime_test_workspace("global-policy-mode");
9519        let thread_root = workspace.join("threads");
9520        let mut builder = ExtensionRegistryBuilder::new();
9521        builder.inference_engine(Arc::new(FakeInferenceEngine));
9522        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
9523            base_path: thread_root.clone(),
9524        }));
9525        let runtime = Runtime::new(
9526            builder.build().unwrap(),
9527            RuntimeConfig {
9528                workspace: Some(workspace.display().to_string()),
9529                ..Default::default()
9530            },
9531        )
9532        .unwrap();
9533
9534        runtime
9535            .set_policy_mode(PolicyMode::AcceptAll, Some("test".to_string()))
9536            .await
9537            .unwrap();
9538
9539        assert!(!thread_root.join("runtime").exists());
9540        let _ = std::fs::remove_dir_all(workspace);
9541    }
9542
9543    #[tokio::test]
9544    async fn task_ledger_enforcement_injects_eval_reminder_before_work() {
9545        let captured = Arc::new(StdMutex::new(None));
9546        let mut builder = ExtensionRegistryBuilder::new();
9547        builder.inference_engine(Arc::new(CapturingEngine {
9548            request: captured.clone(),
9549        }));
9550        builder.tool_contributor(Arc::new(
9551            roder_ext_task_ledger::TaskLedgerToolContributor::default(),
9552        ));
9553        let runtime = Arc::new(
9554            Runtime::new(
9555                builder.build().unwrap(),
9556                RuntimeConfig {
9557                    runtime_profile: RuntimeProfile::Eval,
9558                    policy_mode: PolicyMode::Bypass,
9559                    agent_swarm_mode: false,
9560                    ultra_mode: false,
9561                    ..RuntimeConfig::default()
9562                },
9563            )
9564            .unwrap(),
9565        );
9566        let mut rx = runtime.subscribe_events();
9567        let turn_id = runtime
9568            .start_turn(StartTurnRequest {
9569                thread_id: "thread-ledger".to_string(),
9570                message: "decomposed work".to_string(),
9571                images: Vec::new(),
9572                provider_override: None,
9573                model_override: None,
9574                reasoning_override: None,
9575                workspace: test_workspace(),
9576                instructions: InstructionBundle::default(),
9577                developer_context: None,
9578                task_ledger_required: true,
9579                service_tier_override: None,
9580            })
9581            .await
9582            .unwrap();
9583
9584        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9585            loop {
9586                let envelope = rx.recv().await.unwrap();
9587                if envelope.turn_id.as_deref() == Some(&turn_id)
9588                    && matches!(envelope.event, RoderEvent::TurnCompleted(_))
9589                {
9590                    break;
9591                }
9592            }
9593        })
9594        .await
9595        .unwrap();
9596
9597        let request = captured.lock().unwrap().clone().unwrap();
9598        let developer = request.instructions.developer.unwrap();
9599        assert!(developer.contains("Task Ledger Required"));
9600        assert!(developer.contains("task_ledger.update"));
9601        let tool_names: Vec<_> = request
9602            .tools
9603            .iter()
9604            .map(|tool| tool.name.as_str())
9605            .collect();
9606        assert!(
9607            tool_names.contains(&TASK_LEDGER_TOOL_NAME),
9608            "tool names: {tool_names:?}"
9609        );
9610        assert_eq!(
9611            request.tool_choice,
9612            ToolChoice::Specific(TASK_LEDGER_TOOL_NAME.to_string())
9613        );
9614        assert_eq!(request.tools.len(), 1);
9615        assert_eq!(request.tools[0].name, TASK_LEDGER_TOOL_NAME);
9616    }
9617
9618    #[tokio::test]
9619    async fn eval_task_ledger_blocks_final_answer_until_open_items_are_completed() {
9620        let requests = Arc::new(StdMutex::new(Vec::new()));
9621        let mut builder = ExtensionRegistryBuilder::new();
9622        builder.inference_engine(Arc::new(TaskLedgerCompletionGateEngine {
9623            calls: StdMutex::new(0),
9624            requests: requests.clone(),
9625        }));
9626        builder.tool_contributor(Arc::new(
9627            roder_ext_task_ledger::TaskLedgerToolContributor::default(),
9628        ));
9629        let runtime = Arc::new(
9630            Runtime::new(
9631                builder.build().unwrap(),
9632                RuntimeConfig {
9633                    runtime_profile: RuntimeProfile::Eval,
9634                    policy_mode: PolicyMode::Bypass,
9635                    agent_swarm_mode: false,
9636                    ultra_mode: false,
9637                    ..RuntimeConfig::default()
9638                },
9639            )
9640            .unwrap(),
9641        );
9642        let mut rx = runtime.subscribe_events();
9643        let turn_id = runtime
9644            .start_turn(StartTurnRequest {
9645                thread_id: "thread-ledger-completion".to_string(),
9646                message: "write the answer file".to_string(),
9647                images: Vec::new(),
9648                provider_override: None,
9649                model_override: None,
9650                reasoning_override: None,
9651                workspace: test_workspace(),
9652                instructions: InstructionBundle::default(),
9653                developer_context: None,
9654                task_ledger_required: true,
9655                service_tier_override: None,
9656            })
9657            .await
9658            .unwrap();
9659
9660        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9661            loop {
9662                let envelope = rx.recv().await.unwrap();
9663                if envelope.turn_id.as_deref() != Some(&turn_id) {
9664                    continue;
9665                }
9666                match envelope.event {
9667                    RoderEvent::TurnCompleted(_) => break,
9668                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9669                    _ => {}
9670                }
9671            }
9672        })
9673        .await
9674        .unwrap();
9675
9676        let requests = requests.lock().unwrap().clone();
9677        assert_eq!(requests.len(), 4);
9678        assert!(requests[2].transcript.iter().any(|item| {
9679            matches!(
9680                item,
9681                TranscriptItem::UserMessage(message)
9682                    if message.text.contains("Task Ledger Completion Required")
9683                        && message.text.contains("Write /app/result.txt")
9684            )
9685        }));
9686        assert!(requests[3].transcript.iter().any(|item| {
9687            matches!(
9688                item,
9689                TranscriptItem::ToolResult(result)
9690                    if result.name.as_deref() == Some(TASK_LEDGER_TOOL_NAME)
9691                        && result.result.contains("Task ledger: 2/2 completed")
9692            )
9693        }));
9694    }
9695
9696    #[tokio::test]
9697    async fn eval_task_ledger_checkpoint_requests_scoreable_file_before_final_reserve() {
9698        let requests = Arc::new(StdMutex::new(Vec::new()));
9699        let mut builder = ExtensionRegistryBuilder::new();
9700        builder.inference_engine(Arc::new(TaskLedgerCompletionGateEngine {
9701            calls: StdMutex::new(0),
9702            requests: requests.clone(),
9703        }));
9704        builder.tool_contributor(Arc::new(
9705            roder_ext_task_ledger::TaskLedgerToolContributor::default(),
9706        ));
9707        let runtime = Arc::new(
9708            Runtime::new(
9709                builder.build().unwrap(),
9710                RuntimeConfig {
9711                    runtime_profile: RuntimeProfile::Eval,
9712                    policy_mode: PolicyMode::Bypass,
9713                    agent_swarm_mode: false,
9714                    ultra_mode: false,
9715                    turn_deadline_seconds: Some(120),
9716                    ..RuntimeConfig::default()
9717                },
9718            )
9719            .unwrap(),
9720        );
9721        let mut rx = runtime.subscribe_events();
9722        let turn_id = runtime
9723            .start_turn(StartTurnRequest {
9724                thread_id: "thread-ledger-checkpoint".to_string(),
9725                message: "write the answer file".to_string(),
9726                images: Vec::new(),
9727                provider_override: None,
9728                model_override: None,
9729                reasoning_override: None,
9730                workspace: test_workspace(),
9731                instructions: InstructionBundle::default(),
9732                developer_context: None,
9733                task_ledger_required: true,
9734                service_tier_override: None,
9735            })
9736            .await
9737            .unwrap();
9738
9739        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9740            loop {
9741                let envelope = rx.recv().await.unwrap();
9742                if envelope.turn_id.as_deref() != Some(&turn_id) {
9743                    continue;
9744                }
9745                match envelope.event {
9746                    RoderEvent::TurnCompleted(_) => break,
9747                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9748                    _ => {}
9749                }
9750            }
9751        })
9752        .await
9753        .unwrap();
9754
9755        let requests = requests.lock().unwrap().clone();
9756        assert!(requests.len() >= 2);
9757        assert!(requests[1].transcript.iter().any(|item| {
9758            matches!(
9759                item,
9760                TranscriptItem::UserMessage(message)
9761                    if message.text.contains("Scoreable Output Checkpoint")
9762                        && message.text.contains("ensure the required output file(s) exist")
9763                        && message.text.contains("Write /app/result.txt")
9764            )
9765        }));
9766    }
9767
9768    #[test]
9769    fn deadline_task_ledger_prompt_preserves_scoreable_work_instruction() {
9770        let prompt = task_ledger_deadline_completion_prompt(
9771            12,
9772            30,
9773            "Task Ledger Completion Required: write /app/result.txt, then call task_ledger.update",
9774        );
9775
9776        assert!(prompt.contains("12 seconds remain"));
9777        assert!(prompt.contains("create or update the required scoreable output files"));
9778        assert!(prompt.contains("write /app/result.txt"));
9779        assert!(prompt.contains(TASK_LEDGER_TOOL_NAME));
9780    }
9781
9782    #[test]
9783    fn scoreable_checkpoint_prompt_preserves_provisional_file_instruction() {
9784        let prompt = task_ledger_scoreable_checkpoint_prompt(
9785            120,
9786            "Task Ledger Completion Required: write /app/result.txt, then call task_ledger.update",
9787        );
9788
9789        assert!(prompt.contains("120 seconds remain"));
9790        assert!(prompt.contains("best evidence-backed answer"));
9791        assert!(prompt.contains("even if provisional"));
9792        assert!(prompt.contains("preserve that candidate"));
9793        assert!(prompt.contains("partial-coverage"));
9794        assert!(prompt.contains("write /app/result.txt"));
9795        assert!(prompt.contains(TASK_LEDGER_TOOL_NAME));
9796    }
9797
9798    #[test]
9799    fn open_task_ledger_moves_inference_timeout_to_scoreable_checkpoint() {
9800        let deadline = Some(OffsetDateTime::now_utc() + Duration::seconds(870));
9801        let transcript = vec![TranscriptItem::ToolResult(ToolResultRecord {
9802            id: "ledger-open".to_string(),
9803            name: Some(TASK_LEDGER_TOOL_NAME.to_string()),
9804            result: "Task ledger: 0/1 completed\n- pending: Write /app/result.txt [write]"
9805                .to_string(),
9806            display_payload: None,
9807            is_error: false,
9808        })];
9809
9810        let (_, action) = inference_timeout_deadline(
9811            deadline,
9812            RuntimeProfile::Eval,
9813            true,
9814            30,
9815            false,
9816            0,
9817            &transcript,
9818        )
9819        .unwrap();
9820
9821        assert_eq!(action, InferenceTimeoutAction::ScoreableCheckpoint);
9822    }
9823
9824    #[tokio::test]
9825    async fn verification_gate_forces_eval_code_changes_through_review() {
9826        let mut builder = ExtensionRegistryBuilder::new();
9827        builder.inference_engine(Arc::new(VerificationGateEngine {
9828            calls: StdMutex::new(0),
9829        }));
9830        builder.tool_contributor(Arc::new(WriteFileContributor));
9831        builder.tool_contributor(Arc::new(
9832            roder_ext_verification::VerificationToolContributor,
9833        ));
9834        let runtime = Arc::new(
9835            Runtime::new(
9836                builder.build().unwrap(),
9837                RuntimeConfig {
9838                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9839                    default_model: "mock".to_string(),
9840                    runtime_profile: RuntimeProfile::Eval,
9841                    policy_mode: PolicyMode::Bypass,
9842                    agent_swarm_mode: false,
9843                    ultra_mode: false,
9844                    ..RuntimeConfig::default()
9845                },
9846            )
9847            .unwrap(),
9848        );
9849        let mut rx = runtime.subscribe_events();
9850        let turn_id = runtime
9851            .start_turn(StartTurnRequest {
9852                thread_id: "thread-verification".to_string(),
9853                message: "write code".to_string(),
9854                images: Vec::new(),
9855                provider_override: None,
9856                model_override: None,
9857                reasoning_override: None,
9858                workspace: test_workspace(),
9859                instructions: InstructionBundle::default(),
9860                developer_context: None,
9861                task_ledger_required: false,
9862                service_tier_override: None,
9863            })
9864            .await
9865            .unwrap();
9866
9867        let mut saw_required = false;
9868        let mut saw_completed = false;
9869        let mut final_text = String::new();
9870        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9871            loop {
9872                let envelope = rx.recv().await.unwrap();
9873                if envelope.turn_id.as_deref() != Some(&turn_id) {
9874                    continue;
9875                }
9876                match envelope.event {
9877                    RoderEvent::VerificationRequired(event) => {
9878                        saw_required = true;
9879                        assert_eq!(event.changed_files, vec!["src/lib.rs"]);
9880                    }
9881                    RoderEvent::VerificationCompleted(event) => {
9882                        saw_completed = true;
9883                        assert!(event.passed);
9884                    }
9885                    RoderEvent::InferenceEventReceived(event) => {
9886                        if let InferenceEvent::MessageDelta(delta) = event.event {
9887                            final_text.push_str(&delta.text);
9888                        }
9889                    }
9890                    RoderEvent::TurnCompleted(_) => break,
9891                    _ => {}
9892                }
9893            }
9894        })
9895        .await
9896        .unwrap();
9897
9898        assert!(saw_required);
9899        assert!(saw_completed);
9900        assert!(final_text.contains("verified final"));
9901    }
9902
9903    #[tokio::test]
9904    async fn speed_policy_changes_reasoning_across_eval_model_calls_without_model_switch() {
9905        let requests = Arc::new(StdMutex::new(Vec::new()));
9906        let mut builder = ExtensionRegistryBuilder::new();
9907        builder.inference_engine(Arc::new(SpeedPolicyEngine {
9908            calls: StdMutex::new(0),
9909            requests: requests.clone(),
9910        }));
9911        builder.tool_contributor(Arc::new(WriteFileContributor));
9912        builder.tool_contributor(Arc::new(
9913            roder_ext_verification::VerificationToolContributor,
9914        ));
9915        let runtime = Arc::new(
9916            Runtime::new(
9917                builder.build().unwrap(),
9918                RuntimeConfig {
9919                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9920                    default_model: "gpt-5.5".to_string(),
9921                    runtime_profile: RuntimeProfile::Eval,
9922                    policy_mode: PolicyMode::Bypass,
9923                    agent_swarm_mode: false,
9924                    ultra_mode: false,
9925                    ..RuntimeConfig::default()
9926                },
9927            )
9928            .unwrap(),
9929        );
9930        let mut rx = runtime.subscribe_events();
9931        let turn_id = runtime
9932            .start_turn(StartTurnRequest {
9933                thread_id: "thread-speed-policy".to_string(),
9934                message: "write code".to_string(),
9935                images: Vec::new(),
9936                provider_override: None,
9937                model_override: None,
9938                reasoning_override: None,
9939                workspace: test_workspace(),
9940                instructions: InstructionBundle::default(),
9941                developer_context: None,
9942                task_ledger_required: false,
9943                service_tier_override: None,
9944            })
9945            .await
9946            .unwrap();
9947
9948        let mut saw_speed_policy_event = false;
9949        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9950            loop {
9951                let envelope = rx.recv().await.unwrap();
9952                if envelope.turn_id.as_deref() != Some(&turn_id) {
9953                    continue;
9954                }
9955                match envelope.event {
9956                    RoderEvent::InferenceStarted(event) => {
9957                        if event.speed_policy.is_some() {
9958                            saw_speed_policy_event = true;
9959                        }
9960                    }
9961                    RoderEvent::TurnCompleted(_) => break,
9962                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9963                    _ => {}
9964                }
9965            }
9966        })
9967        .await
9968        .unwrap();
9969
9970        let requests = requests.lock().unwrap().clone();
9971        assert!(saw_speed_policy_event);
9972        assert!(requests.len() >= 4);
9973        assert!(requests.iter().all(|request| {
9974            request.model.provider == roder_api::catalog::PROVIDER_MOCK
9975                && request.model.model == "gpt-5.5"
9976        }));
9977        assert_eq!(
9978            requests[0].runtime.speed_policy.as_ref().map(|d| d.phase),
9979            Some(roder_api::inference::SpeedPolicyPhase::Orientation)
9980        );
9981        assert_eq!(requests[0].reasoning.level.as_deref(), Some(REASONING_HIGH));
9982        assert_eq!(
9983            requests[1].runtime.speed_policy.as_ref().map(|d| d.phase),
9984            Some(roder_api::inference::SpeedPolicyPhase::Execution)
9985        );
9986        assert_eq!(requests[1].reasoning.level.as_deref(), Some(REASONING_LOW));
9987        assert_eq!(
9988            requests[2].runtime.speed_policy.as_ref().map(|d| d.phase),
9989            Some(roder_api::inference::SpeedPolicyPhase::Verification)
9990        );
9991        assert_eq!(requests[2].reasoning.level.as_deref(), Some(REASONING_HIGH));
9992        assert_eq!(
9993            requests[2]
9994                .metadata
9995                .pointer("/speedPolicy/phase")
9996                .and_then(serde_json::Value::as_str),
9997            Some("verification")
9998        );
9999    }
10000
10001    #[tokio::test]
10002    async fn deadline_turn_timeout_emits_partial_result_and_clears_active_turn() {
10003        let mut builder = ExtensionRegistryBuilder::new();
10004        builder.inference_engine(Arc::new(DeadlineEngine));
10005        let runtime = Arc::new(
10006            Runtime::new(
10007                builder.build().unwrap(),
10008                RuntimeConfig {
10009                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
10010                    default_model: "mock".to_string(),
10011                    runtime_profile: RuntimeProfile::Eval,
10012                    turn_deadline_seconds: Some(1),
10013                    ..RuntimeConfig::default()
10014                },
10015            )
10016            .unwrap(),
10017        );
10018        let mut rx = runtime.subscribe_events();
10019        let turn_id = runtime
10020            .start_turn(StartTurnRequest {
10021                thread_id: "thread-deadline".to_string(),
10022                message: "slow work".to_string(),
10023                images: Vec::new(),
10024                provider_override: None,
10025                model_override: None,
10026                reasoning_override: None,
10027                workspace: test_workspace(),
10028                instructions: InstructionBundle::default(),
10029                developer_context: None,
10030                task_ledger_required: false,
10031                service_tier_override: None,
10032            })
10033            .await
10034            .unwrap();
10035
10036        let mut saw_partial = false;
10037        let mut saw_deadline = false;
10038        let mut failed_kind = None;
10039        tokio::time::timeout(std::time::Duration::from_secs(5), async {
10040            loop {
10041                let envelope = rx.recv().await.unwrap();
10042                if envelope.turn_id.as_deref() != Some(&turn_id) {
10043                    continue;
10044                }
10045                match envelope.event {
10046                    RoderEvent::TurnPartialResult(event) => {
10047                        saw_partial = event.summary.contains("partial turn state");
10048                    }
10049                    RoderEvent::TurnDeadlineExceeded(event) => {
10050                        saw_deadline = event.partial_result.contains("transcript items");
10051                    }
10052                    RoderEvent::TurnFailed(event) => {
10053                        failed_kind = event.error_kind;
10054                        break;
10055                    }
10056                    _ => {}
10057                }
10058            }
10059        })
10060        .await
10061        .unwrap();
10062
10063        for _ in 0..20 {
10064            if !runtime.active_turns.read().await.contains_key(&turn_id) {
10065                break;
10066            }
10067            tokio::time::sleep(std::time::Duration::from_millis(10)).await;
10068        }
10069        assert!(saw_partial);
10070        assert!(saw_deadline);
10071        assert_eq!(failed_kind.as_deref(), Some("deadline_timeout"));
10072        assert!(!runtime.active_turns.read().await.contains_key(&turn_id));
10073    }
10074
10075    #[tokio::test]
10076    async fn deadline_skips_subagent_task_when_remaining_budget_is_too_low() {
10077        let calls = Arc::new(StdMutex::new(0));
10078        let mut builder = ExtensionRegistryBuilder::new();
10079        builder.inference_engine(Arc::new(CapturingEngine {
10080            request: Arc::new(StdMutex::new(None)),
10081        }));
10082        let task_tool = Arc::new(CountingTaskTool {
10083            calls: calls.clone(),
10084        });
10085        builder.tool_contributor(Arc::new(TestToolContributor { tool: task_tool }));
10086        let runtime = Arc::new(
10087            Runtime::new(
10088                builder.build().unwrap(),
10089                RuntimeConfig {
10090                    policy_mode: PolicyMode::Bypass,
10091                    agent_swarm_mode: false,
10092                    ultra_mode: false,
10093                    ..RuntimeConfig::default()
10094                },
10095            )
10096            .unwrap(),
10097        );
10098
10099        let result = runtime
10100            .route_tool_call(
10101                &"thread-deadline-task".to_string(),
10102                &"turn-deadline-task".to_string(),
10103                ToolCallCompleted {
10104                    id: "task-1".to_string(),
10105                    name: "task".to_string(),
10106                    arguments: serde_json::json!({
10107                        "description": "inspect",
10108                        "prompt": "read"
10109                    })
10110                    .to_string(),
10111                },
10112                None,
10113                Some(OffsetDateTime::now_utc() + Duration::seconds(1)),
10114            )
10115            .await
10116            .unwrap();
10117
10118        assert!(result.is_error);
10119        assert!(result.result.contains("deadline policy skipped"));
10120        assert_eq!(*calls.lock().unwrap(), 0);
10121    }
10122
10123    struct TestToolContributor {
10124        tool: Arc<dyn ToolExecutor>,
10125    }
10126
10127    impl ToolContributor for TestToolContributor {
10128        fn id(&self) -> String {
10129            "test-tool".to_string()
10130        }
10131
10132        fn contribute(&self, registry: &mut ToolRegistry) -> anyhow::Result<()> {
10133            registry.register(self.tool.clone())
10134        }
10135    }
10136
10137    #[tokio::test]
10138    async fn agent_swarm_mode_override_is_per_thread() {
10139        let runtime = Runtime::fake().unwrap();
10140        let trigger = roder_api::subagents::AgentSwarmModeTrigger::Manual;
10141
10142        // Off everywhere by default.
10143        assert!(
10144            !runtime
10145                .effective_agent_swarm_mode_for_thread("thread-a")
10146                .await
10147        );
10148        assert!(
10149            !runtime
10150                .effective_agent_swarm_mode_for_thread("thread-b")
10151                .await
10152        );
10153
10154        // Enabling on thread-a does not leak into thread-b.
10155        assert!(
10156            runtime
10157                .set_agent_swarm_mode_for_thread("thread-a", true, trigger)
10158                .await
10159        );
10160        assert!(
10161            runtime
10162                .effective_agent_swarm_mode_for_thread("thread-a")
10163                .await
10164        );
10165        assert!(
10166            !runtime
10167                .effective_agent_swarm_mode_for_thread("thread-b")
10168                .await
10169        );
10170
10171        // A per-thread `off` override wins over a runtime-global `on` default.
10172        runtime.set_agent_swarm_mode(true, trigger).await.unwrap();
10173        runtime
10174            .set_agent_swarm_mode_for_thread("thread-b", false, trigger)
10175            .await;
10176        assert!(
10177            !runtime
10178                .effective_agent_swarm_mode_for_thread("thread-b")
10179                .await,
10180            "explicit per-thread off overrides the global on default"
10181        );
10182        // A thread with no override still follows the global default.
10183        assert!(
10184            runtime
10185                .effective_agent_swarm_mode_for_thread("thread-c")
10186                .await,
10187            "threads without an override follow the runtime-global default"
10188        );
10189    }
10190
10191    #[tokio::test]
10192    async fn set_agent_swarm_mode_for_thread_emits_event_with_real_thread_id() {
10193        let runtime = Runtime::fake().unwrap();
10194        let mut events = runtime.subscribe_events();
10195        runtime
10196            .set_agent_swarm_mode_for_thread(
10197                "thread-xyz",
10198                true,
10199                roder_api::subagents::AgentSwarmModeTrigger::Task,
10200            )
10201            .await;
10202        let mut saw = false;
10203        for _ in 0..8 {
10204            let envelope = tokio::time::timeout(std::time::Duration::from_secs(2), events.recv())
10205                .await
10206                .unwrap()
10207                .unwrap();
10208            if let RoderEvent::AgentSwarmModeChanged(event) = envelope.event {
10209                assert_eq!(event.thread_id, "thread-xyz");
10210                assert!(event.enabled);
10211                assert_eq!(
10212                    event.trigger,
10213                    roder_api::subagents::AgentSwarmModeTrigger::Task
10214                );
10215                saw = true;
10216                break;
10217            }
10218        }
10219        assert!(
10220            saw,
10221            "expected an AgentSwarmModeChanged event for the thread"
10222        );
10223    }
10224
10225    #[tokio::test]
10226    async fn ultra_mode_override_is_per_thread() {
10227        let runtime = Runtime::fake().unwrap();
10228        let trigger = roder_api::subagents::UltraModeTrigger::Manual;
10229
10230        assert!(!runtime.effective_ultra_mode_for_thread("thread-a").await);
10231        assert!(!runtime.effective_ultra_mode_for_thread("thread-b").await);
10232
10233        assert!(
10234            runtime
10235                .set_ultra_mode_for_thread("thread-a", true, trigger)
10236                .await
10237        );
10238        assert!(runtime.effective_ultra_mode_for_thread("thread-a").await);
10239        assert!(!runtime.effective_ultra_mode_for_thread("thread-b").await);
10240
10241        runtime.set_ultra_mode(true, trigger).await.unwrap();
10242        runtime
10243            .set_ultra_mode_for_thread("thread-b", false, trigger)
10244            .await;
10245        assert!(
10246            !runtime.effective_ultra_mode_for_thread("thread-b").await,
10247            "explicit per-thread off overrides the global on default"
10248        );
10249        assert!(
10250            runtime.effective_ultra_mode_for_thread("thread-c").await,
10251            "threads without an override follow the runtime-global default"
10252        );
10253    }
10254
10255    #[tokio::test]
10256    async fn set_ultra_mode_for_thread_emits_event_with_real_thread_id() {
10257        let runtime = Runtime::fake().unwrap();
10258        let mut events = runtime.subscribe_events();
10259        runtime
10260            .set_ultra_mode_for_thread(
10261                "thread-ultra",
10262                true,
10263                roder_api::subagents::UltraModeTrigger::Task,
10264            )
10265            .await;
10266        let mut saw = false;
10267        for _ in 0..8 {
10268            let envelope = tokio::time::timeout(std::time::Duration::from_secs(2), events.recv())
10269                .await
10270                .unwrap()
10271                .unwrap();
10272            if let RoderEvent::UltraModeChanged(event) = envelope.event {
10273                assert_eq!(event.thread_id, "thread-ultra");
10274                assert!(event.enabled);
10275                assert_eq!(event.trigger, roder_api::subagents::UltraModeTrigger::Task);
10276                saw = true;
10277                break;
10278            }
10279        }
10280        assert!(saw, "expected an UltraModeChanged event for the thread");
10281    }
10282}
10283
10284#[cfg(test)]
10285#[path = "runtime/mailbox_test_helpers.rs"]
10286mod mailbox_test_helpers;