Skip to main content

roder_core/
runtime.rs

1#[path = "runtime/sampling.rs"]
2pub(crate) mod sampling;
3#[path = "tool_advertisement.rs"]
4mod tool_advertisement;
5use sampling::{
6    SamplingPreempted, SamplingRetry, completed_call_replayed, preemptible, wait_for_steer,
7};
8#[path = "runtime/sampling_output.rs"]
9mod sampling_output;
10use sampling_output::{OutputContext, SamplingOutput};
11mod compaction_template;
12#[path = "runtime/eager_tools.rs"]
13mod eager_tools;
14mod thread_admission;
15mod owner_handoff;
16use eager_tools::EagerTools;
17#[path = "runtime/patch_progress.rs"]
18mod patch_progress;
19use patch_progress::PatchProgressTracker;
20
21use std::collections::{HashMap, HashSet};
22use std::path::PathBuf;
23use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
24use std::sync::{Arc, Weak};
25
26use anyhow::Context;
27use futures::StreamExt;
28use futures::future::{AbortHandle, Abortable, BoxFuture, try_join_all};
29use roder_api::catalog::{
30    EDIT_TOOL_EDIT, EDIT_TOOL_PATCH, PROVIDER_GEMINI, REASONING_NONE, REASONING_ULTRA,
31    built_in_model_profile, built_in_model_profile_for_provider, lookup_model,
32    model_supports_reasoning_effort,
33};
34use roder_api::context::PolicyGate;
35use roder_api::events::*;
36use roder_api::extension::ExtensionRegistry;
37use roder_api::inference::{
38    AgentInferenceRequest, HostedWebSearchConfig, HostedWebSearchMode, InferenceEngine,
39    InferenceEvent, InferenceTurnContext, InstructionBundle, ModelHarnessProfile,
40    ModelSchemaPolicy, ModelSelection, OutputConfig, ProviderTurnCleanup, ReasoningConfig,
41    RuntimeHints, RuntimeProfile, TokenUsage, ToolCallCompleted, ToolSearchConfig,
42    ToolSearchConfigOverlay, finish_reason_from_stop_reason,
43};
44use roder_api::inference_routing::{InferenceRoutingOutcome, ModelSelectionMode};
45use roder_api::lifecycle::{
46    LifecycleMetricsSnapshot, TurnCleanupOwnership, TurnCleanupState, TurnLifecycleReason,
47    TurnLifecycleRecord, TurnLifecycleSnapshot, TurnLifecycleState, turn_lifecycle_snapshot,
48};
49use roder_api::policy_mode::{PolicyDecision, PolicyMode};
50use roder_api::reliability::{
51    ReliabilityContext, ReliabilityDetails, ReliabilityErrorClass, ReliabilityLimitDecision,
52    ReliabilityLimitRecorded, ReliabilityRequestPolicy, ReliabilityRetryDecision,
53    ReliabilityRetryRecorded, provider_retry_delay_ms,
54};
55use roder_api::remote_runner::{
56    RemoteRunnerProvider, RemoteRunnerSession, RemoteWorkspace, RunnerDestination,
57    RunnerSessionState, ThreadRunnerBinding,
58};
59use roder_api::subagents::SubagentDefinition;
60use roder_api::teams::{
61    TeamId, TeamMailboxMessage, TeamMailboxMessageKind, TeamMemberDescriptor, TeamMemberRole,
62    TeamMemberStatus,
63};
64use roder_api::thread::{
65    ThreadItemEvent, ThreadItemEventKind, ThreadMetadata, ThreadSnapshot, ThreadStore,
66    ThreadUsageMetadata, is_synthetic_event_thread_id, validate_thread_workspace,
67};
68use roder_api::tools::{ToolCall, ToolChoice, ToolExecutionContext, ToolRegistry, ToolResult};
69use roder_api::transcript::{
70    AssistantMessage, ErrorRecord, InputImage, ReasoningSummary, ToolCallRecord, ToolResultRecord,
71    TranscriptItem, UserMessage,
72};
73use roder_sandbox::ScopedFilesystem;
74use roder_sandbox::process::LocalProcessRunner;
75use roder_skills::{SkillRegistry, SkillRegistryOptions};
76use time::{Duration, OffsetDateTime};
77use tokio::sync::{Mutex, Notify, RwLock, oneshot};
78
79mod codex_v2;
80
81use crate::artifacts::{
82    ContextArtifactStore as FilesystemContextArtifactStore, default_context_artifact_dir,
83};
84use crate::bus::EventBus;
85use crate::dynamic_workflows::{
86    DynamicWorkflowEffortProfile, RuntimeDynamicWorkflowConfig, WorkflowTriggerDecision,
87    classify_workflow_trigger, ultracode_reasoning_level_for_model,
88};
89use crate::fake_provider::FakeInferenceEngine;
90use crate::goals::RuntimeGoalController;
91use crate::inference_routing::{
92    InferenceRoutingRequest, RuntimeInferenceRouterConfig, collect_inference_routing_candidates,
93    route_inference_selection, transcript_failure_count_since,
94};
95use crate::instructions::{
96    apply_agent_swarm_mode, apply_codex_multi_agent_mode, apply_model_instruction_overlay,
97    apply_parallel_web_tools, apply_plan_mode, apply_runtime_profile, apply_task_ledger_required,
98    apply_thread_developer_instructions, apply_turn_developer_context,
99};
100use crate::policy_gate::DefaultPolicyGate;
101use crate::reliability::{
102    ReliabilityLimitHit, RuntimeReliabilityConfig, TurnReliabilityState,
103    provider_stream_retry_cause,
104};
105pub use crate::speed_policy::RuntimeSpeedPolicyConfig;
106use crate::speed_policy::{SpeedPolicyState, reasoning_from_decision};
107use crate::subagent_traces::{RuntimeAgentSwarmProgressSink, RuntimeSubagentTraceSink};
108use crate::teams::{TeamManager, TeamMemberStartRequest, TeamStartRequest, TeamState};
109use crate::thread_item_cache::{ThreadItemCache, ThreadItemCacheEntry};
110use crate::verification_gate::VerificationGateState;
111
112const MAX_TOOL_ROUNDS_PER_TURN: usize = 1024;
113/// Injected when `continue_on_failure_limit` turns a consecutive-tool-failure
114/// limit into a continuation, or as the empty-tool-call persistence nudge, so
115/// the model keeps working instead of ending the turn early.
116const RELIABILITY_CONTINUATION_PROMPT: &str = "Verify the task is fully complete. If any part remains unfinished or unverified, keep working using the available tools — try an alternative approach if one is failing. Only stop once the solution is complete and verified; if it is already complete, restate your final answer.";
117/// Capacity of the runtime event broadcast ring buffer. The TUI drains this on
118/// its render loop, which during an active turn only wakes at the ~6 FPS status
119/// animation cadence (backend events do not wake the input poll), so up to
120/// ~166ms of events buffer between drains. A reasoning-high provider that runs
121/// its whole tool loop inside one turn (e.g. claude-code) streams a firehose of
122/// `InferenceEventReceived` deltas plus per-step tool/thinking events; the old
123/// 1024-slot buffer overflowed in those windows and silently dropped tool and
124/// thinking rows from the live view. Sized for generous headroom across bursts
125/// and brief render stalls.
126const EVENT_BUS_CAPACITY: usize = 16_384;
127
128/// Turn id reported to session-scoped local hooks. `SessionStart` and
129/// `SessionEnd` bracket the session rather than any one turn, but the hook
130/// payload carries a turn id, so they share this synthetic value.
131const SESSION_HOOK_TURN_ID: &str = "session";
132pub(crate) const FINAL_ANSWER_PHASE: &str = "final_answer";
133pub(crate) const TASK_LEDGER_TOOL_NAME: &str = "task_ledger.update";
134const TASK_LEDGER_COMPLETION_REMINDER_LIMIT: u8 = 2;
135const TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS: u64 = 180;
136const TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT: u8 = 1;
137pub(crate) const MIN_CHILD_DEADLINE_SECONDS: u64 = 2;
138const MODEL_PROFILE_TRACE_KIND: &str = "model_profile_segment";
139const MODEL_SWITCH_SUMMARY_PREFIX: &str = "Model switch summary:";
140const PROVIDER_CLEANUP_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5);
141
142#[derive(Clone, Copy, Debug, Eq, PartialEq)]
143enum InferenceTimeoutAction {
144    ScoreableCheckpoint,
145    Finalization,
146}
147
148#[derive(Debug, Clone)]
149pub struct RuntimeConfig {
150    pub default_provider: String,
151    pub default_model: String,
152    pub reasoning: Option<String>,
153    pub auto_compact_token_limit: Option<u32>,
154    pub file_backed_dynamic_context: bool,
155    pub hosted_web_search: HostedWebSearchConfig,
156    pub tool_search: ToolSearchConfig,
157    pub provider_tool_search: HashMap<String, ToolSearchConfigOverlay>,
158    pub model_tool_search: HashMap<String, ToolSearchConfigOverlay>,
159    pub model_edit_tools: HashMap<String, String>,
160    pub model_parallel_tool_calls: HashMap<String, bool>,
161    pub model_profiles: HashMap<String, ModelHarnessProfile>,
162    pub tool_allowlist: Vec<String>,
163    /// Seconds a host-executed external tool call may stay unresolved before it fails with a timeout error.
164    pub external_tool_timeout_seconds: u64,
165    pub command_shell: String,
166    pub workspace: Option<String>,
167    pub policy_mode: PolicyMode,
168    /// Whether agent-swarm mode is active (roadmap 104). When on, the runtime
169    /// injects the swarm reminder into each turn's developer instructions so any
170    /// client benefits, not just the TUI.
171    pub agent_swarm_mode: bool,
172    /// Whether ultra mode is active. When on, the runtime injects proactive
173    /// multi-agent delegation policy for any model (not only Codex Sol/Terra
174    /// Ultra effort). Selecting Ultra reasoning on Sol/Terra still enables
175    /// proactive multi-agent for that model without requiring this flag.
176    pub ultra_mode: bool,
177    pub runtime_profile: RuntimeProfile,
178    pub inference_router: RuntimeInferenceRouterConfig,
179    pub speed_policy: RuntimeSpeedPolicyConfig,
180    pub dynamic_workflows: RuntimeDynamicWorkflowConfig,
181    pub reliability: RuntimeReliabilityConfig,
182    /// Positive wall-clock limit for a turn in any runtime profile. `None` or zero is unbounded.
183    pub turn_deadline_seconds: Option<u64>,
184    pub remote_runner_destination: Option<RunnerDestination>,
185    pub team_data_dir: Option<PathBuf>,
186    pub roadmap_data_dir: Option<PathBuf>,
187    pub media_generation: crate::media_generation::RuntimeMediaGenerationConfig,
188    /// `[review]` settings: the review sub-turn and its publishers.
189    pub review: crate::review::RuntimeReviewConfig,
190}
191
192impl Default for RuntimeConfig {
193    fn default() -> Self {
194        Self {
195            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
196            default_model: "mock".to_string(),
197            reasoning: None,
198            auto_compact_token_limit: None,
199            file_backed_dynamic_context: true,
200            hosted_web_search: HostedWebSearchConfig::cached(),
201            tool_search: ToolSearchConfig::default(),
202            provider_tool_search: HashMap::new(),
203            model_tool_search: HashMap::new(),
204            model_edit_tools: HashMap::new(),
205            model_parallel_tool_calls: HashMap::new(),
206            model_profiles: HashMap::new(),
207            tool_allowlist: Vec::new(),
208            external_tool_timeout_seconds: DEFAULT_EXTERNAL_TOOL_TIMEOUT_SECONDS,
209            command_shell: roder_api::command_shell::default_command_shell(),
210            workspace: None,
211            policy_mode: PolicyMode::Default,
212            agent_swarm_mode: false,
213            ultra_mode: false,
214            runtime_profile: RuntimeProfile::Interactive,
215            inference_router: RuntimeInferenceRouterConfig::default(),
216            speed_policy: RuntimeSpeedPolicyConfig::default(),
217            dynamic_workflows: RuntimeDynamicWorkflowConfig::default(),
218            reliability: RuntimeReliabilityConfig::default(),
219            turn_deadline_seconds: None,
220            remote_runner_destination: None,
221            team_data_dir: None,
222            roadmap_data_dir: None,
223            media_generation: crate::media_generation::RuntimeMediaGenerationConfig::default(),
224            review: crate::review::RuntimeReviewConfig::default(),
225        }
226    }
227}
228
229#[derive(Debug, Clone)]
230pub struct StartTurnRequest {
231    pub thread_id: ThreadId,
232    pub message: String,
233    pub images: Vec<InputImage>,
234    pub provider_override: Option<String>,
235    pub model_override: Option<String>,
236    pub reasoning_override: Option<String>,
237    pub workspace: String,
238    pub instructions: InstructionBundle,
239    /**
240     * Per-turn developer-authority context for this turn's InstructionBundle.
241     * Applies to every inference round of the turn, is never written to
242     * thread state, and does not carry over to later turns.
243     */
244    pub developer_context: Option<String>,
245    pub task_ledger_required: bool,
246    /**
247     * Per-turn provider service tier (OpenAI `service_tier`, e.g.
248     * `"priority"` for Fast mode). Carried to every inference round of the
249     * turn as `RuntimeHints::service_tier`; `None` keeps the provider
250     * default. Providers without a tier concept ignore it.
251     */
252    pub service_tier_override: Option<String>,
253}
254
255#[derive(Debug, Clone)]
256pub struct CreateThreadRequest {
257    pub title: Option<String>,
258    pub workspace: String,
259    pub workspace_id: Option<String>,
260    pub root_id: Option<String>,
261    pub provider: Option<String>,
262    pub model: Option<String>,
263    pub selection_mode: Option<ModelSelectionMode>,
264    /// Per-thread tool filter applied on top of the runtime allowlist. Empty = no filtering.
265    pub tool_allowlist: Vec<String>,
266    /// Host-supplied instructions added to the developer slot of every turn's inference request.
267    pub developer_instructions: Option<String>,
268    /// Host-executed tool specs advertised to the model on every turn of this thread.
269    pub external_tools: Vec<roder_api::tools::ToolSpec>,
270    /// Explicit remote-runner binding for the thread's native coding tools.
271    pub runner: Option<ThreadRunnerSelection>,
272}
273
274/**
275 * Thread-level remote-runner selection. The destination config is persisted
276 * with the thread, so secrets must reach the provider through its
277 * environment, not this config.
278 */
279#[derive(Debug, Clone)]
280pub struct ThreadRunnerSelection {
281    pub provider_id: String,
282    pub config: serde_json::Value,
283    /// Absolute path on the runner used as the thread's coding-tool workspace root.
284    pub workspace: String,
285    /**
286     * Extra absolute runner paths file reads may resolve under, beyond
287     * `workspace`. Writes and the working directory stay confined to
288     * `workspace`.
289     */
290    pub read_roots: Vec<String>,
291}
292
293#[derive(Debug, Clone, PartialEq, Eq)]
294pub struct PendingPlanExit {
295    pub thread_id: ThreadId,
296    pub turn_id: TurnId,
297    pub request_id: String,
298    pub target_mode: PolicyMode,
299    pub plan_summary: Option<String>,
300    pub next_steps: Vec<String>,
301    pub requested_at: OffsetDateTime,
302    pub expires_at: Option<OffsetDateTime>,
303}
304
305pub(crate) struct PendingToolApproval {
306    pub(crate) thread_id: ThreadId,
307    pub(crate) turn_id: TurnId,
308    pub(crate) tool_id: String,
309    pub(crate) tool_name: String,
310    pub(crate) call: roder_api::tools::ToolCall,
311    pub(crate) tx: oneshot::Sender<bool>,
312}
313
314pub(crate) struct PendingUserInput {
315    pub(crate) thread_id: ThreadId,
316    pub(crate) turn_id: TurnId,
317    pub(crate) tx: oneshot::Sender<serde_json::Value>,
318}
319
320/// Host answer to an external tool call delivered via `tools/resolve`.
321#[derive(Debug, Clone, PartialEq, Eq)]
322pub struct ExternalToolResolution {
323    pub output: String,
324    pub is_error: bool,
325}
326
327pub(crate) struct PendingExternalToolCall {
328    pub(crate) thread_id: ThreadId,
329    pub(crate) turn_id: TurnId,
330    pub(crate) tool_id: String,
331    pub(crate) tool_name: String,
332    pub(crate) tx: oneshot::Sender<ExternalToolResolution>,
333}
334
335#[derive(Clone)]
336struct ActiveTurnHandle {
337    thread_id: ThreadId,
338    abort: AbortHandle,
339    steers: Arc<Mutex<Vec<QueuedTurnSteer>>>,
340    steer_changed: tokio::sync::watch::Sender<u64>,
341    drain: Arc<TurnDrainHandle>,
342}
343
344/// Tracks a task after it has been interrupted and removed from the interactive
345/// active-turn map. This lets users start a follow-up turn immediately while a
346/// bounded runtime shutdown can still await the original task's cleanup.
347struct TurnDrainHandle {
348    thread_id: ThreadId,
349    interrupt_requested: AtomicBool,
350    interrupt_reason: Mutex<Option<TurnLifecycleReason>>,
351    completed: AtomicBool,
352    completed_notify: Notify,
353}
354
355/// Result of a bounded runtime drain. `Clean` proves the runtime's own turn
356/// tasks have reached their cleanup paths; provider-specific child-process
357/// reaping remains an explicit provider capability.
358#[derive(Debug, Clone, PartialEq, Eq)]
359pub enum RuntimeDrainOutcome {
360    Clean {
361        interrupted_turn_ids: Vec<TurnId>,
362    },
363    DeadlineExceeded {
364        interrupted_turn_ids: Vec<TurnId>,
365        remaining_turn_ids: Vec<TurnId>,
366    },
367    /// At least one lifecycle transition initiated by this drain could not be
368    /// durably appended. The runtime still made its best cleanup attempt, but
369    /// callers must not claim a persisted terminal state as proof of recovery.
370    PersistenceFailed {
371        interrupted_turn_ids: Vec<TurnId>,
372        remaining_turn_ids: Vec<TurnId>,
373    },
374}
375
376#[derive(Clone)]
377struct QueuedTurnSteer {
378    message: UserMessage,
379    mailbox_ack: Option<MailboxDeliveryAck>,
380}
381
382#[derive(Clone)]
383struct MailboxDeliveryAck {
384    team_id: TeamId,
385    message_ids: Vec<String>,
386}
387
388#[derive(Clone)]
389struct InheritedTurnContext {
390    workspace: String,
391    instructions: InstructionBundle,
392    developer_context: Option<String>,
393}
394
395#[derive(Debug, Clone, Default, PartialEq, Eq)]
396pub struct ThreadActivity {
397    pub active_turn_id: Option<TurnId>,
398    pub active_flags: Vec<String>,
399}
400
401/// Per-thread settings persisted at thread creation and applied to every turn.
402#[derive(Debug, Clone, Default)]
403pub(crate) struct ThreadTurnOverrides {
404    pub(crate) tool_allowlist: Vec<String>,
405    pub(crate) developer_instructions: Option<String>,
406    pub(crate) external_tools: Vec<roder_api::tools::ToolSpec>,
407}
408
409#[derive(Debug, Clone, Copy, PartialEq, Eq)]
410pub(crate) enum TurnRunOutcome {
411    Completed,
412    Stopped,
413}
414
415impl PendingPlanExit {
416    pub fn new(
417        thread_id: ThreadId,
418        turn_id: TurnId,
419        request_id: String,
420        target_mode: PolicyMode,
421        plan_summary: Option<String>,
422        next_steps: Vec<String>,
423    ) -> Self {
424        let requested_at = OffsetDateTime::now_utc();
425        Self {
426            thread_id,
427            turn_id,
428            request_id,
429            target_mode,
430            plan_summary,
431            next_steps,
432            requested_at,
433            expires_at: Some(requested_at + default_plan_exit_timeout()),
434        }
435    }
436
437    pub fn is_expired(&self, now: OffsetDateTime) -> bool {
438        self.expires_at.is_some_and(|expires_at| now >= expires_at)
439    }
440}
441
442pub fn default_plan_exit_timeout() -> Duration {
443    Duration::minutes(10)
444}
445
446pub const DEFAULT_EXTERNAL_TOOL_TIMEOUT_SECONDS: u64 = 300;
447
448fn format_mailbox_messages(team: &TeamState, messages: &[TeamMailboxMessage]) -> String {
449    messages
450        .iter()
451        .map(|message| {
452            let recipient = team
453                .members
454                .iter()
455                .find(|member| member.id == message.to_member_id)
456                .map(canonical_team_member_path)
457                .unwrap_or_else(|| format!("/root/{}", message.to_member_id));
458            let sender = message
459                .from_member_id
460                .as_deref()
461                .and_then(|id| team.members.iter().find(|member| member.id == id))
462                .map(canonical_team_member_path)
463                .unwrap_or_else(|| "/root".to_string());
464            let message_type = match message.kind {
465                TeamMailboxMessageKind::Message => "MESSAGE",
466                TeamMailboxMessageKind::NewTask => "NEW_TASK",
467                TeamMailboxMessageKind::FinalAnswer => "FINAL_ANSWER",
468            };
469            format!(
470                "Message Type: {message_type}\nTask name: {recipient}\nSender: {sender}\nPayload:\n{}",
471                message.text
472            )
473        })
474        .collect::<Vec<_>>()
475        .join("\n\n")
476}
477
478fn canonical_team_member_path(member: &TeamMemberDescriptor) -> String {
479    if let Some(agent_path) = member
480        .agent_path
481        .as_deref()
482        .filter(|path| *path == "/root" || path.starts_with("/root/"))
483    {
484        return agent_path.to_string();
485    }
486    if member.role == TeamMemberRole::Lead {
487        return "/root".to_string();
488    }
489    member
490        .task_name
491        .as_deref()
492        .filter(|name| !name.trim().is_empty())
493        .map(|name| format!("/root/{}", name.trim().trim_matches('/')))
494        .unwrap_or_else(|| format!("/root/{}", member.id))
495}
496
497fn is_codex_v2_team(team: &TeamState) -> bool {
498    team.members.iter().any(|member| {
499        member
500            .model
501            .as_deref()
502            .is_some_and(|model| model_supports_reasoning_effort(model, REASONING_ULTRA))
503    })
504}
505
506fn runtime_local_team_view(
507    mut team: TeamState,
508    active_thread_ids: &std::collections::HashSet<ThreadId>,
509) -> TeamState {
510    for member in &mut team.members {
511        if member.status == TeamMemberStatus::Running
512            && !active_thread_ids.contains(&member.thread_id)
513        {
514            member.status = TeamMemberStatus::Interrupted;
515            member.current_turn_id = None;
516        }
517    }
518    team
519}
520
521type RunnerToolExecutionKey = (String, String);
522type RunnerToolExecutionMutex = Mutex<()>;
523type RunnerToolExecutionLockRegistry =
524    Mutex<HashMap<RunnerToolExecutionKey, Weak<RunnerToolExecutionMutex>>>;
525
526pub struct Runtime {
527    pub bus: EventBus,
528    pub registry: ExtensionRegistry,
529    config: RwLock<RuntimeConfig>,
530    pub(crate) execution_lease: Option<Arc<crate::execution_lease::RuntimeExecutionLease>>,
531    pending_plan_exit: RwLock<Option<PendingPlanExit>>,
532    pub(crate) pending_tool_approvals: Mutex<HashMap<String, PendingToolApproval>>,
533    pub(crate) pending_user_inputs: Mutex<HashMap<String, PendingUserInput>>,
534    pub(crate) pending_external_tool_calls: Mutex<HashMap<String, PendingExternalToolCall>>,
535    /// Serializes turn admission with shutdown quiescing. A drain takes this
536    /// gate before flipping `accepting_turns` and snapshotting active work so a
537    /// turn that observed the old flag cannot be inserted after the snapshot.
538    turn_admission: Mutex<()>,
539    active_turns: RwLock<HashMap<TurnId, ActiveTurnHandle>>,
540    turn_drains: RwLock<HashMap<TurnId, Arc<TurnDrainHandle>>>,
541    /// Provider cleanup handles register synchronously through the inference
542    /// tool-executor context. They are intentionally separate from active turn
543    /// admission so an interrupted turn can remain drainable after a follow-up
544    /// turn starts on the same thread.
545    provider_turn_cleanups: std::sync::Mutex<HashMap<TurnId, Arc<dyn ProviderTurnCleanup>>>,
546    accepting_turns: AtomicBool,
547    /// Monotonic counter used by bounded drains to surface lifecycle-state
548    /// persistence failures without making an individual provider turn fail.
549    lifecycle_persistence_failures: AtomicUsize,
550    lifecycle_shutdown_drains: AtomicUsize,
551    lifecycle_clean_shutdowns: AtomicUsize,
552    lifecycle_deadline_exceeded: AtomicUsize,
553    lifecycle_persistence_failed_drains: AtomicUsize,
554    lifecycle_restart_reconciliations: AtomicUsize,
555    lifecycle_shutdown_drain_duration_ms_total: AtomicUsize,
556    provider_cleanup_confirmed: AtomicUsize,
557    provider_cleanup_timed_out: AtomicUsize,
558    provider_cleanup_unknown: AtomicUsize,
559    active_turns_changed: Notify,
560    active_turn_selections: RwLock<HashMap<TurnId, ModelSelectionMode>>,
561    compaction_templates: RwLock<HashMap<ThreadId, AgentInferenceRequest>>,
562    thread_admission_gates: Mutex<HashMap<ThreadId, Arc<Mutex<()>>>>,
563    /// Threads that have already dispatched a `SessionStart` local hook.
564    ///
565    /// `ThreadCreated` fires once, but `ThreadLoaded` fires on every read of the
566    /// thread from its store, so without this a session ran its setup hooks
567    /// several times — including after the turn had already stopped.
568    session_hook_started: RwLock<HashSet<ThreadId>>,
569    active_turn_contexts: RwLock<HashMap<TurnId, InheritedTurnContext>>,
570    /// Process-local execution boundary. Local/CLI runtimes default to true;
571    /// hosted runtime pools set this false so missing runner metadata fails
572    /// closed instead of exposing the host workspace.
573    allow_local_workspaces: AtomicBool,
574    // Spawn-time live authority retained for every reusable turn of a long-lived teammate.
575    // This stays process-local because developer_context is explicitly volatile and must never
576    // be persisted to thread state.
577    team_member_turn_contexts: Mutex<HashMap<ThreadId, InheritedTurnContext>>,
578    workspace: PathBuf,
579    pub(crate) teams: TeamManager,
580    agent_team_spawn_lock: Mutex<()>,
581    pub(crate) roadmaps: Mutex<roder_roadmap::RoadmapRuntime>,
582    /// Completed reviews, most recent last, so a later publish can resolve a
583    /// review id back to its findings. Bounded; see `review::run`.
584    pub(crate) reviews: Mutex<Vec<crate::review::ReviewRecord>>,
585    pub(crate) goals: Arc<RuntimeGoalController>,
586    context_artifacts: roder_api::artifacts::ContextArtifactStore,
587    pub(crate) thread_store: Option<Arc<dyn ThreadStore>>,
588    thread_item_cache: Mutex<ThreadItemCache>,
589    pub(crate) tool_registry: ToolRegistry,
590    media_generation: Arc<crate::media_generation::MediaGenerationService>,
591    pub(crate) skills: RwLock<SkillRegistry>,
592    /// Lazily-started bounded dispatch of emitted events to registry
593    /// `EventSink`s (process extensions etc.); see `event_sink_dispatch`.
594    event_sink_dispatcher: tokio::sync::OnceCell<crate::event_sink_dispatch::EventSinkDispatcher>,
595    pub(crate) compaction_hysteresis:
596        std::sync::Mutex<HashMap<ThreadId, crate::compaction_runtime::CompactionState>>,
597    /// Per-thread agent-swarm mode overrides (roadmap 104). A thread present in
598    /// this map uses its stored value; absent threads fall back to the
599    /// runtime-global `RuntimeConfig.agent_swarm_mode` default. This mirrors the
600    /// team per-member policy-mode override idiom so swarm mode is per-thread
601    /// like the other `thread/*` operations, without a separate runtime.
602    agent_swarm_modes: RwLock<HashMap<ThreadId, bool>>,
603    /// Per-thread ultra mode overrides. A thread present in this map uses its
604    /// stored value; absent threads fall back to the runtime-global
605    /// `RuntimeConfig.ultra_mode` default.
606    ultra_modes: RwLock<HashMap<ThreadId, bool>>,
607    /**
608     * Live remote-runner sessions keyed by thread. Each per-thread mutex is
609     * held across provider initialization, making the first workspace-tool
610     * access a singleflight while allowing unrelated threads to initialize in
611     * parallel. Failed initialization is shared with current waiters, then
612     * evicted so a later tool call can retry.
613     */
614    runner_sessions: Arc<Mutex<HashMap<ThreadId, Arc<RunnerSessionSlot>>>>,
615    /**
616     * Outer tool-call locks keyed by the actual remote session, not the
617     * per-thread destination id. Hosted runtimes are tenant-scoped, so this
618     * serializes threads that share one sandbox without coupling different
619     * tenants or independent runner sessions.
620     */
621    runner_tool_execution_locks: RunnerToolExecutionLockRegistry,
622}
623
624#[derive(Clone)]
625struct CachedRunnerSession {
626    destination: RunnerDestination,
627    session: Arc<dyn RemoteRunnerSession>,
628}
629
630struct RunnerSessionSlot {
631    provider_id: String,
632    destination_id: String,
633    state: Mutex<RunnerSessionSlotState>,
634    initialized: Notify,
635}
636
637enum RunnerSessionSlotState {
638    Empty,
639    Initializing,
640    Ready(CachedRunnerSession),
641    Failed(Arc<str>),
642}
643
644impl RunnerSessionSlot {
645    fn empty(destination: &RunnerDestination) -> Self {
646        Self {
647            provider_id: destination.provider_id.clone(),
648            destination_id: destination.id.clone(),
649            state: Mutex::new(RunnerSessionSlotState::Empty),
650            initialized: Notify::new(),
651        }
652    }
653
654    fn ready(session: CachedRunnerSession) -> Self {
655        Self {
656            provider_id: session.destination.provider_id.clone(),
657            destination_id: session.destination.id.clone(),
658            state: Mutex::new(RunnerSessionSlotState::Ready(session)),
659            initialized: Notify::new(),
660        }
661    }
662
663    fn matches(&self, destination: &RunnerDestination) -> bool {
664        self.provider_id == destination.provider_id && self.destination_id == destination.id
665    }
666}
667
668impl Runtime {
669    pub fn new(registry: ExtensionRegistry, config: RuntimeConfig) -> anyhow::Result<Self> {
670        if registry.inference_engines.is_empty() {
671            anyhow::bail!("at least one inference engine must be registered");
672        }
673        validate_runtime_config_reasoning(&config)?;
674        validate_runtime_inference_router_config(&registry, &config)?;
675
676        let bus = EventBus::new(EVENT_BUS_CAPACITY);
677        let thread_store = registry
678            .thread_stores
679            .first()
680            .map(|factory| factory.create());
681        let mut tool_registry = ToolRegistry::default();
682        for contributor in &registry.tools {
683            contributor
684                .contribute(&mut tool_registry)
685                .with_context(|| format!("tool contributor {} failed", contributor.id()))?;
686        }
687        crate::agent_control_tools::contribute_agent_control_tools(&mut tool_registry)?;
688
689        let media_generation = Arc::new(crate::media_generation::MediaGenerationService::new(
690            registry.media_generator_providers.clone(),
691            config.media_generation.clone(),
692        ));
693        tool_registry.replace(Arc::new(
694            crate::media_generation::MediaGenerateImageTool::new(media_generation.clone()),
695        ));
696
697        let team_data_dir = config.team_data_dir.clone();
698        let workspace = config
699            .workspace
700            .clone()
701            .map(PathBuf::from)
702            .unwrap_or(std::env::current_dir()?);
703        let roadmap_data_dir = config
704            .roadmap_data_dir
705            .clone()
706            .unwrap_or_else(|| workspace.join(".roder"));
707        let context_artifacts = thread_store
708            .as_ref()
709            .and_then(|store| store.context_artifact_store())
710            .or_else(|| {
711                thread_store
712                    .as_ref()
713                    .and_then(|store| store.local_thread_root())
714                    .map(FilesystemContextArtifactStore::shared_thread_scoped)
715            })
716            .unwrap_or_else(|| {
717                FilesystemContextArtifactStore::shared_legacy(default_context_artifact_dir())
718            });
719        let goals = Arc::new(RuntimeGoalController::new(
720            bus.clone(),
721            thread_store.clone(),
722        ));
723        let runtime = Self {
724            bus,
725            registry,
726            config: RwLock::new(config),
727            execution_lease: None,
728            pending_plan_exit: RwLock::new(None),
729            pending_tool_approvals: Mutex::new(HashMap::new()),
730            pending_user_inputs: Mutex::new(HashMap::new()),
731            pending_external_tool_calls: Mutex::new(HashMap::new()),
732            turn_admission: Mutex::new(()),
733            active_turns: RwLock::new(HashMap::new()),
734            turn_drains: RwLock::new(HashMap::new()),
735            provider_turn_cleanups: std::sync::Mutex::new(HashMap::new()),
736            accepting_turns: AtomicBool::new(true),
737            lifecycle_persistence_failures: AtomicUsize::new(0),
738            lifecycle_shutdown_drains: AtomicUsize::new(0),
739            lifecycle_clean_shutdowns: AtomicUsize::new(0),
740            lifecycle_deadline_exceeded: AtomicUsize::new(0),
741            lifecycle_persistence_failed_drains: AtomicUsize::new(0),
742            lifecycle_restart_reconciliations: AtomicUsize::new(0),
743            lifecycle_shutdown_drain_duration_ms_total: AtomicUsize::new(0),
744            provider_cleanup_confirmed: AtomicUsize::new(0),
745            provider_cleanup_timed_out: AtomicUsize::new(0),
746            provider_cleanup_unknown: AtomicUsize::new(0),
747            active_turns_changed: Notify::new(),
748            active_turn_selections: RwLock::new(HashMap::new()),
749            compaction_templates: RwLock::new(HashMap::new()),
750            thread_admission_gates: Mutex::new(HashMap::new()),
751            session_hook_started: RwLock::new(HashSet::new()),
752            active_turn_contexts: RwLock::new(HashMap::new()),
753            allow_local_workspaces: AtomicBool::new(true),
754            team_member_turn_contexts: Mutex::new(HashMap::new()),
755            workspace: workspace.clone(),
756            teams: TeamManager::new(
757                team_data_dir.unwrap_or_else(crate::teams::default_team_data_dir),
758            ),
759            agent_team_spawn_lock: Mutex::new(()),
760            roadmaps: Mutex::new(roder_roadmap::RoadmapRuntime::new(
761                workspace,
762                roadmap_data_dir,
763            )),
764            reviews: Mutex::new(Vec::new()),
765            goals,
766            context_artifacts,
767            thread_store,
768            thread_item_cache: Mutex::new(ThreadItemCache::default()),
769            tool_registry,
770            media_generation,
771            skills: RwLock::new(SkillRegistry::load(SkillRegistryOptions::new(
772                PathBuf::new(),
773            ))),
774            event_sink_dispatcher: tokio::sync::OnceCell::new(),
775            compaction_hysteresis: crate::compaction_runtime::compaction_hysteresis_state(),
776            agent_swarm_modes: RwLock::new(HashMap::new()),
777            ultra_modes: RwLock::new(HashMap::new()),
778            runner_sessions: Arc::new(Mutex::new(HashMap::new())),
779            runner_tool_execution_locks: Mutex::new(HashMap::new()),
780        };
781        runtime.bus.emit(RoderEvent::RuntimeStarted(RuntimeStarted {
782            timestamp: OffsetDateTime::now_utc(),
783        }));
784        for manifest in &runtime.registry.manifests {
785            runtime
786                .bus
787                .emit(RoderEvent::ExtensionRegistered(ExtensionRegistered {
788                    extension_id: manifest.id.clone(),
789                    timestamp: OffsetDateTime::now_utc(),
790                }));
791        }
792        Ok(runtime)
793    }
794
795    pub fn from_engine(engine: Arc<dyn InferenceEngine>) -> anyhow::Result<Self> {
796        let mut builder = roder_api::extension::ExtensionRegistryBuilder::new();
797        builder.inference_engine(engine);
798        Self::new(builder.build()?, RuntimeConfig::default())
799    }
800
801    pub fn fake() -> anyhow::Result<Self> {
802        Self::from_engine(Arc::new(FakeInferenceEngine))
803    }
804
805    pub fn subscribe_events(&self) -> tokio::sync::broadcast::Receiver<EventEnvelope> {
806        self.bus.subscribe()
807    }
808
809    /// Returns process-local, redacted lifecycle health counters. The snapshot
810    /// intentionally has fixed fields and contains no provider, command,
811    /// process, thread, or turn identifiers.
812    pub fn lifecycle_metrics(&self) -> LifecycleMetricsSnapshot {
813        LifecycleMetricsSnapshot {
814            shutdown_drain_count: self.lifecycle_shutdown_drains.load(Ordering::Acquire) as u64,
815            clean_shutdown_count: self.lifecycle_clean_shutdowns.load(Ordering::Acquire) as u64,
816            deadline_exceeded_count: self.lifecycle_deadline_exceeded.load(Ordering::Acquire)
817                as u64,
818            persistence_failed_count: self
819                .lifecycle_persistence_failed_drains
820                .load(Ordering::Acquire) as u64,
821            restart_reconciliation_count: self
822                .lifecycle_restart_reconciliations
823                .load(Ordering::Acquire) as u64,
824            lifecycle_persistence_failure_count: self
825                .lifecycle_persistence_failures
826                .load(Ordering::Acquire) as u64,
827            shutdown_drain_duration_ms_total: self
828                .lifecycle_shutdown_drain_duration_ms_total
829                .load(Ordering::Acquire) as u64,
830            provider_cleanup_confirmed_count: self
831                .provider_cleanup_confirmed
832                .load(Ordering::Acquire) as u64,
833            provider_cleanup_timed_out_count: self
834                .provider_cleanup_timed_out
835                .load(Ordering::Acquire) as u64,
836            provider_cleanup_unknown_count: self.provider_cleanup_unknown.load(Ordering::Acquire)
837                as u64,
838        }
839    }
840
841    async fn persist_turn_lifecycle_record(&self, record: &TurnLifecycleRecord) -> bool {
842        let Some(store) = &self.thread_store else {
843            return true;
844        };
845        let state = match record.extension_state() {
846            Ok(state) => state,
847            Err(_) => {
848                self.lifecycle_persistence_failures
849                    .fetch_add(1, Ordering::AcqRel);
850                return false;
851            }
852        };
853        // Lifecycle diagnostics must not turn a provider result into a failed
854        // turn merely because a third-party store lacks extension-state support.
855        // JSONL/Postgres stores persist this append-only record atomically at
856        // their own storage boundary.
857        if store
858            .append_extension_state(&record.thread_id, &state)
859            .await
860            .is_err()
861        {
862            self.lifecycle_persistence_failures
863                .fetch_add(1, Ordering::AcqRel);
864            return false;
865        }
866        true
867    }
868
869    async fn record_turn_lifecycle(
870        &self,
871        thread_id: ThreadId,
872        turn_id: TurnId,
873        state: TurnLifecycleState,
874        cleanup: TurnCleanupState,
875        reason: Option<TurnLifecycleReason>,
876    ) -> TurnLifecycleRecord {
877        self.record_turn_lifecycle_with_ownership(
878            thread_id,
879            turn_id,
880            state,
881            cleanup,
882            reason,
883            TurnCleanupOwnership::RuntimeTaskOnly,
884        )
885        .await
886    }
887
888    async fn record_turn_lifecycle_with_ownership(
889        &self,
890        thread_id: ThreadId,
891        turn_id: TurnId,
892        state: TurnLifecycleState,
893        cleanup: TurnCleanupState,
894        reason: Option<TurnLifecycleReason>,
895        ownership: TurnCleanupOwnership,
896    ) -> TurnLifecycleRecord {
897        let record = TurnLifecycleRecord::new(
898            thread_id,
899            turn_id,
900            state,
901            cleanup,
902            reason,
903            OffsetDateTime::now_utc(),
904        )
905        .with_ownership(ownership);
906        let _ = self.persist_turn_lifecycle_record(&record).await;
907        self.emit(RoderEvent::TurnLifecycleUpdated(record.clone()))
908            .await;
909        record
910    }
911
912    pub(crate) fn register_provider_turn_cleanup(
913        &self,
914        turn_id: &TurnId,
915        cleanup: Arc<dyn ProviderTurnCleanup>,
916    ) {
917        if let Ok(mut cleanups) = self.provider_turn_cleanups.lock() {
918            cleanups.insert(turn_id.clone(), cleanup);
919        }
920    }
921
922    fn provider_cleanup_ownership(&self, turn_id: &TurnId) -> TurnCleanupOwnership {
923        self.provider_turn_cleanups
924            .lock()
925            .ok()
926            .and_then(|cleanups| cleanups.get(turn_id).cloned())
927            .map(|cleanup| cleanup.ownership())
928            .unwrap_or(TurnCleanupOwnership::RuntimeTaskOnly)
929    }
930
931    async fn await_provider_turn_cleanup(
932        &self,
933        turn_id: &TurnId,
934    ) -> (TurnCleanupState, TurnCleanupOwnership) {
935        let cleanup = self
936            .provider_turn_cleanups
937            .lock()
938            .ok()
939            .and_then(|mut cleanups| cleanups.remove(turn_id));
940        let Some(cleanup) = cleanup else {
941            return (
942                TurnCleanupState::Unknown,
943                TurnCleanupOwnership::RuntimeTaskOnly,
944            );
945        };
946        let ownership = cleanup.ownership();
947        match tokio::time::timeout(PROVIDER_CLEANUP_TIMEOUT, cleanup.wait_for_cleanup()).await {
948            Ok(Ok(())) => {
949                self.provider_cleanup_confirmed
950                    .fetch_add(1, Ordering::AcqRel);
951                (
952                    TurnCleanupState::Completed,
953                    TurnCleanupOwnership::ProviderCleanupConfirmed,
954                )
955            }
956            Ok(Err(_)) => {
957                self.provider_cleanup_unknown.fetch_add(1, Ordering::AcqRel);
958                (TurnCleanupState::Unknown, ownership)
959            }
960            Err(_) => {
961                self.provider_cleanup_timed_out
962                    .fetch_add(1, Ordering::AcqRel);
963                (TurnCleanupState::TimedOut, ownership)
964            }
965        }
966    }
967
968    fn record_lifecycle_drain_outcome(
969        &self,
970        started_at: tokio::time::Instant,
971        outcome: &RuntimeDrainOutcome,
972    ) {
973        self.lifecycle_shutdown_drains
974            .fetch_add(1, Ordering::AcqRel);
975        self.lifecycle_shutdown_drain_duration_ms_total.fetch_add(
976            started_at.elapsed().as_millis().min(usize::MAX as u128) as usize,
977            Ordering::AcqRel,
978        );
979        match outcome {
980            RuntimeDrainOutcome::Clean { .. } => {
981                self.lifecycle_clean_shutdowns
982                    .fetch_add(1, Ordering::AcqRel);
983            }
984            RuntimeDrainOutcome::DeadlineExceeded { .. } => {
985                self.lifecycle_deadline_exceeded
986                    .fetch_add(1, Ordering::AcqRel);
987            }
988            RuntimeDrainOutcome::PersistenceFailed { .. } => {
989                self.lifecycle_persistence_failed_drains
990                    .fetch_add(1, Ordering::AcqRel);
991            }
992        }
993    }
994
995    fn lifecycle_record_for_event(event: &RoderEvent) -> Option<TurnLifecycleRecord> {
996        match event {
997            RoderEvent::TurnStarted(event) => Some(TurnLifecycleRecord::new(
998                event.thread_id.clone(),
999                event.turn_id.clone(),
1000                TurnLifecycleState::Running,
1001                TurnCleanupState::NotRequested,
1002                None,
1003                event.timestamp,
1004            )),
1005            // `run_turn` persists completed turns after it has awaited a
1006            // registered provider cleanup handle. Do not let the generic event
1007            // projection overwrite that acknowledgement with an unproven state.
1008            RoderEvent::TurnCompleted(_) => None,
1009            // Some failure paths already have an event before the turn wrapper
1010            // reaches its cleanup acknowledgement. Preserve a durable fallback
1011            // reason here; the wrapper appends a later record with the more
1012            // precise cleanup outcome when a provider registered one.
1013            RoderEvent::TurnFailed(event) => Some(TurnLifecycleRecord::new(
1014                event.thread_id.clone(),
1015                event.turn_id.clone(),
1016                TurnLifecycleState::Failed,
1017                TurnCleanupState::Unknown,
1018                Some(if event.error_kind.as_deref() == Some("deadline_timeout") {
1019                    TurnLifecycleReason::DeadlineExceeded
1020                } else {
1021                    TurnLifecycleReason::ProviderFailure
1022                }),
1023                event.timestamp,
1024            )),
1025            RoderEvent::TurnDeadlineExceeded(_) | RoderEvent::TurnInterrupted(_) => None,
1026            RoderEvent::TurnLifecycleUpdated(_) => None,
1027            _ => None,
1028        }
1029    }
1030
1031    pub fn registry(&self) -> &ExtensionRegistry {
1032        &self.registry
1033    }
1034
1035    pub fn media_generation(&self) -> Arc<crate::media_generation::MediaGenerationService> {
1036        self.media_generation.clone()
1037    }
1038
1039    pub fn context_artifacts(&self) -> roder_api::artifacts::ContextArtifactStore {
1040        self.context_artifacts.clone()
1041    }
1042
1043    pub async fn execute_workflow_tool(
1044        &self,
1045        thread_id: ThreadId,
1046        tool_name: &str,
1047        arguments: serde_json::Value,
1048    ) -> anyhow::Result<ToolResult> {
1049        let Some(executor) = self.tool_registry.get(tool_name) else {
1050            anyhow::bail!("tool not found: {tool_name}");
1051        };
1052        let tool_call = ToolCall {
1053            id: format!("slash-{tool_name}"),
1054            name: tool_name.to_string(),
1055            raw_arguments: serde_json::to_string(&arguments)?,
1056            arguments,
1057            thread_id: thread_id.clone(),
1058            turn_id: "slash-command".to_string(),
1059        };
1060        let runtime_config = self.status().await;
1061        let ctx = self.tool_execution_context(
1062            thread_id,
1063            "slash-command".to_string(),
1064            runtime_config.policy_mode,
1065            runtime_config.workspace.as_deref(),
1066            Some(&runtime_config.command_shell),
1067        );
1068        executor.execute(ctx, tool_call).await
1069    }
1070
1071    pub(crate) fn tool_execution_context(
1072        &self,
1073        thread_id: ThreadId,
1074        turn_id: TurnId,
1075        mode: PolicyMode,
1076        workspace: Option<&str>,
1077        command_shell: Option<&str>,
1078    ) -> ToolExecutionContext {
1079        let mut ctx = ToolExecutionContext::new(thread_id, turn_id, mode)
1080            .with_command_shell(command_shell.unwrap_or_default())
1081            .with_process_runner(Arc::new(LocalProcessRunner))
1082            .with_context_artifacts(self.context_artifacts.backend())
1083            .with_goal_controller(self.goals.clone())
1084            .with_subagent_trace_sink(Arc::new(RuntimeSubagentTraceSink::new(
1085                self.bus.clone(),
1086                self.thread_store.clone(),
1087            )))
1088            .with_swarm_progress_sink(Arc::new(RuntimeAgentSwarmProgressSink::new(
1089                self.bus.clone(),
1090                self.thread_store.clone(),
1091            )));
1092        if let Some(workspace) = workspace {
1093            ctx = ctx.with_workspace_handle(Arc::new(ScopedFilesystem::new(workspace)));
1094        }
1095        ctx
1096    }
1097
1098    pub async fn status(&self) -> RuntimeConfig {
1099        self.config.read().await.clone()
1100    }
1101
1102    pub async fn set_skills(&self, skills: SkillRegistry) {
1103        *self.skills.write().await = skills;
1104    }
1105
1106    pub async fn skills_snapshot(&self) -> SkillRegistry {
1107        self.skills.read().await.clone()
1108    }
1109
1110    pub fn workspace(&self) -> PathBuf {
1111        self.workspace.clone()
1112    }
1113
1114    /// Controls whether native workspace tools may execute without an
1115    /// explicit remote-runner binding. Hosted runtimes disable this after
1116    /// construction; ordinary local runtimes retain the compatible default.
1117    pub fn set_allow_local_workspaces(&self, allowed: bool) {
1118        self.allow_local_workspaces
1119            .store(allowed, Ordering::Relaxed);
1120    }
1121
1122    pub fn allows_local_workspaces(&self) -> bool {
1123        self.allow_local_workspaces.load(Ordering::Relaxed)
1124    }
1125
1126    pub async fn set_remote_runner_destination(&self, destination: Option<RunnerDestination>) {
1127        let lifecycle = destination.as_ref().map(|destination| RunnerLifecycle {
1128            destination_id: destination.id.clone(),
1129            provider_id: destination.provider_id.clone(),
1130            state: "configured".to_string(),
1131            session_id: None,
1132            timestamp: OffsetDateTime::now_utc(),
1133        });
1134        self.config.write().await.remote_runner_destination = destination;
1135        if let Some(lifecycle) = lifecycle {
1136            self.emit(RoderEvent::RunnerLifecycle(lifecycle)).await;
1137        } else {
1138            self.emit(RoderEvent::RunnerLifecycle(RunnerLifecycle {
1139                destination_id: "local".to_string(),
1140                provider_id: "local".to_string(),
1141                state: "local_fallback".to_string(),
1142                session_id: None,
1143                timestamp: OffsetDateTime::now_utc(),
1144            }))
1145            .await;
1146        }
1147    }
1148
1149    pub async fn set_file_backed_dynamic_context(&self, enabled: bool) -> RuntimeConfig {
1150        let mut cfg = self.config.write().await;
1151        cfg.file_backed_dynamic_context = enabled;
1152        cfg.clone()
1153    }
1154
1155    pub async fn set_command_shell(&self, shell: String) -> RuntimeConfig {
1156        let mut cfg = self.config.write().await;
1157        cfg.command_shell = shell;
1158        cfg.clone()
1159    }
1160
1161    pub async fn pending_plan_exit(&self) -> Option<PendingPlanExit> {
1162        let mut pending = self.pending_plan_exit.write().await;
1163        let current = pending.clone()?;
1164        if !current.is_expired(OffsetDateTime::now_utc()) {
1165            return Some(current);
1166        }
1167        *pending = None;
1168        drop(pending);
1169        self.emit_plan_exit_resolved(&current, false, self.status().await.policy_mode)
1170            .await;
1171        None
1172    }
1173
1174    pub async fn set_policy_mode(
1175        &self,
1176        mode: PolicyMode,
1177        reason: Option<String>,
1178    ) -> anyhow::Result<RuntimeConfig> {
1179        let mut cfg = self.config.write().await;
1180        let previous_mode = cfg.policy_mode;
1181        cfg.policy_mode = mode;
1182        let next = cfg.clone();
1183        drop(cfg);
1184        self.emit(RoderEvent::PolicyModeChanged(PolicyModeChanged {
1185            thread_id: "runtime".to_string(),
1186            turn_id: None,
1187            previous_mode,
1188            new_mode: mode,
1189            reason,
1190            timestamp: OffsetDateTime::now_utc(),
1191        }))
1192        .await;
1193        self.auto_resolve_pending_tool_approvals_for_mode(mode)
1194            .await;
1195        Ok(next)
1196    }
1197
1198    /// Toggle agent-swarm mode for the runtime (roadmap 104). When enabled, the
1199    /// swarm reminder is injected into each turn's developer instructions so
1200    /// every app-server/SDK client gets it, not only the TUI.
1201    pub async fn set_agent_swarm_mode(
1202        &self,
1203        enabled: bool,
1204        trigger: roder_api::subagents::AgentSwarmModeTrigger,
1205    ) -> anyhow::Result<RuntimeConfig> {
1206        let mut cfg = self.config.write().await;
1207        cfg.agent_swarm_mode = enabled;
1208        let next = cfg.clone();
1209        drop(cfg);
1210        self.emit(RoderEvent::AgentSwarmModeChanged(
1211            roder_api::subagents::AgentSwarmModeChanged {
1212                thread_id: "runtime".to_string(),
1213                turn_id: None,
1214                enabled,
1215                trigger,
1216                timestamp: OffsetDateTime::now_utc(),
1217            },
1218        ))
1219        .await;
1220        Ok(next)
1221    }
1222
1223    /// Toggle agent-swarm mode for a single thread (roadmap 104). The override is
1224    /// stored per thread so toggling swarm mode on one thread does not leak the
1225    /// reminder into other threads sharing the runtime; absent threads fall back
1226    /// to the runtime-global default. The emitted `AgentSwarmModeChanged` event
1227    /// carries the real `thread_id`.
1228    pub async fn set_agent_swarm_mode_for_thread(
1229        &self,
1230        thread_id: &str,
1231        enabled: bool,
1232        trigger: roder_api::subagents::AgentSwarmModeTrigger,
1233    ) -> bool {
1234        {
1235            let mut modes = self.agent_swarm_modes.write().await;
1236            modes.insert(thread_id.to_string(), enabled);
1237        }
1238        self.emit(RoderEvent::AgentSwarmModeChanged(
1239            roder_api::subagents::AgentSwarmModeChanged {
1240                thread_id: thread_id.to_string(),
1241                turn_id: None,
1242                enabled,
1243                trigger,
1244                timestamp: OffsetDateTime::now_utc(),
1245            },
1246        ))
1247        .await;
1248        enabled
1249    }
1250
1251    /// Resolve whether agent-swarm mode is active for `thread_id`: the per-thread
1252    /// override when present, otherwise the runtime-global default. The turn loop
1253    /// uses this to decide whether to inject the swarm reminder.
1254    pub async fn effective_agent_swarm_mode_for_thread(&self, thread_id: &str) -> bool {
1255        if let Some(enabled) = self.agent_swarm_modes.read().await.get(thread_id).copied() {
1256            return enabled;
1257        }
1258        self.status().await.agent_swarm_mode
1259    }
1260
1261    /// Toggle ultra mode for the runtime. When enabled, proactive multi-agent
1262    /// policy is injected into each turn's developer instructions for any
1263    /// model, so every app-server/SDK client gets it (not only the TUI).
1264    pub async fn set_ultra_mode(
1265        &self,
1266        enabled: bool,
1267        trigger: roder_api::subagents::UltraModeTrigger,
1268    ) -> anyhow::Result<RuntimeConfig> {
1269        let mut cfg = self.config.write().await;
1270        cfg.ultra_mode = enabled;
1271        let next = cfg.clone();
1272        drop(cfg);
1273        self.emit(RoderEvent::UltraModeChanged(
1274            roder_api::subagents::UltraModeChanged {
1275                thread_id: "runtime".to_string(),
1276                turn_id: None,
1277                enabled,
1278                trigger,
1279                timestamp: OffsetDateTime::now_utc(),
1280            },
1281        ))
1282        .await;
1283        Ok(next)
1284    }
1285
1286    /// Toggle ultra mode for a single thread. The override is stored per thread
1287    /// so toggling on one thread does not leak proactive multi-agent policy into
1288    /// other threads sharing the runtime; absent threads fall back to the
1289    /// runtime-global default.
1290    pub async fn set_ultra_mode_for_thread(
1291        &self,
1292        thread_id: &str,
1293        enabled: bool,
1294        trigger: roder_api::subagents::UltraModeTrigger,
1295    ) -> bool {
1296        {
1297            let mut modes = self.ultra_modes.write().await;
1298            modes.insert(thread_id.to_string(), enabled);
1299        }
1300        self.emit(RoderEvent::UltraModeChanged(
1301            roder_api::subagents::UltraModeChanged {
1302                thread_id: thread_id.to_string(),
1303                turn_id: None,
1304                enabled,
1305                trigger,
1306                timestamp: OffsetDateTime::now_utc(),
1307            },
1308        ))
1309        .await;
1310        enabled
1311    }
1312
1313    /// Resolve whether ultra mode is active for `thread_id`: the per-thread
1314    /// override when present, otherwise the runtime-global default. The turn
1315    /// loop uses this (together with Sol/Terra Ultra effort) to decide whether
1316    /// to inject proactive multi-agent policy.
1317    pub async fn effective_ultra_mode_for_thread(&self, thread_id: &str) -> bool {
1318        if let Some(enabled) = self.ultra_modes.read().await.get(thread_id).copied() {
1319            return enabled;
1320        }
1321        self.status().await.ultra_mode
1322    }
1323
1324    pub async fn set_hosted_web_search(
1325        &self,
1326        mode: HostedWebSearchMode,
1327    ) -> anyhow::Result<RuntimeConfig> {
1328        let mut cfg = self.config.write().await;
1329        cfg.hosted_web_search = HostedWebSearchConfig { mode };
1330        Ok(cfg.clone())
1331    }
1332
1333    async fn auto_resolve_pending_tool_approvals_for_mode(&self, mode: PolicyMode) {
1334        let gate = DefaultPolicyGate::new();
1335        let mut pending = self.pending_tool_approvals.lock().await;
1336        let approval_ids = pending
1337            .iter()
1338            .filter_map(|(approval_id, approval)| {
1339                let ctx = ToolExecutionContext::new(
1340                    approval.thread_id.clone(),
1341                    approval.turn_id.clone(),
1342                    mode,
1343                );
1344                matches!(
1345                    gate.decide(&approval.call, mode, &ctx),
1346                    PolicyDecision::AutoApproved { .. }
1347                )
1348                .then_some(approval_id.clone())
1349            })
1350            .collect::<Vec<_>>();
1351        let approvals = approval_ids
1352            .into_iter()
1353            .filter_map(|approval_id| {
1354                pending
1355                    .remove(&approval_id)
1356                    .map(|approval| (approval_id, approval))
1357            })
1358            .collect::<Vec<_>>();
1359        drop(pending);
1360
1361        for (approval_id, approval) in approvals {
1362            let ctx = ToolExecutionContext::new(
1363                approval.thread_id.clone(),
1364                approval.turn_id.clone(),
1365                mode,
1366            );
1367            let decision = gate.decide(&approval.call, mode, &ctx);
1368            self.emit(RoderEvent::PolicyDecisionRecorded(PolicyDecisionRecorded {
1369                thread_id: approval.thread_id.clone(),
1370                turn_id: approval.turn_id.clone(),
1371                tool_id: approval.tool_id.clone(),
1372                tool_name: approval.tool_name.clone(),
1373                mode,
1374                decision,
1375                timestamp: OffsetDateTime::now_utc(),
1376            }))
1377            .await;
1378            if mode == PolicyMode::Bypass {
1379                self.emit(RoderEvent::PolicyBypassActive(PolicyBypassActive {
1380                    thread_id: approval.thread_id.clone(),
1381                    turn_id: approval.turn_id.clone(),
1382                    tool_id: approval.tool_id.clone(),
1383                    tool_name: approval.tool_name.clone(),
1384                    timestamp: OffsetDateTime::now_utc(),
1385                }))
1386                .await;
1387            }
1388            self.emit(RoderEvent::ApprovalResolved(ApprovalResolved {
1389                thread_id: approval.thread_id,
1390                turn_id: approval.turn_id,
1391                approval_id,
1392                tool_id: approval.tool_id,
1393                tool_name: approval.tool_name,
1394                approved: true,
1395                timestamp: OffsetDateTime::now_utc(),
1396            }))
1397            .await;
1398            let _ = approval.tx.send(true);
1399        }
1400    }
1401
1402    pub async fn record_pending_plan_exit(&self, pending: PendingPlanExit) {
1403        *self.pending_plan_exit.write().await = Some(pending.clone());
1404        self.emit(RoderEvent::PolicyExitPlanRequested(
1405            PolicyExitPlanRequested {
1406                thread_id: pending.thread_id,
1407                turn_id: pending.turn_id,
1408                request_id: pending.request_id,
1409                target_mode: pending.target_mode,
1410                plan_summary: pending.plan_summary,
1411                next_steps: pending.next_steps,
1412                timestamp: OffsetDateTime::now_utc(),
1413            },
1414        ))
1415        .await;
1416    }
1417
1418    pub async fn resolve_pending_plan_exit(
1419        &self,
1420        request_id: &str,
1421        approved: bool,
1422    ) -> anyhow::Result<Option<PendingPlanExit>> {
1423        let mut pending = self.pending_plan_exit.write().await;
1424        let Some(current) = pending.clone() else {
1425            return Ok(None);
1426        };
1427        if current.request_id != request_id {
1428            anyhow::bail!("pending plan exit request {request_id:?} was not found");
1429        }
1430        *pending = None;
1431        drop(pending);
1432
1433        let approved = approved && !current.is_expired(OffsetDateTime::now_utc());
1434        let resolved_mode = if approved {
1435            let mut cfg = self.config.write().await;
1436            let previous_mode = cfg.policy_mode;
1437            cfg.policy_mode = current.target_mode;
1438            drop(cfg);
1439            self.emit(RoderEvent::PolicyModeChanged(PolicyModeChanged {
1440                thread_id: current.thread_id.clone(),
1441                turn_id: Some(current.turn_id.clone()),
1442                previous_mode,
1443                new_mode: current.target_mode,
1444                reason: Some("approved plan exit".to_string()),
1445                timestamp: OffsetDateTime::now_utc(),
1446            }))
1447            .await;
1448            self.auto_resolve_pending_tool_approvals_for_mode(current.target_mode)
1449                .await;
1450            current.target_mode
1451        } else {
1452            self.status().await.policy_mode
1453        };
1454        self.emit_plan_exit_resolved(&current, approved, resolved_mode)
1455            .await;
1456        Ok(Some(current))
1457    }
1458
1459    pub async fn resolve_tool_approval(
1460        &self,
1461        approval_id: &str,
1462        approved: bool,
1463    ) -> anyhow::Result<bool> {
1464        let pending = self.pending_tool_approvals.lock().await.remove(approval_id);
1465        let Some(pending) = pending else {
1466            return Ok(false);
1467        };
1468        self.emit(RoderEvent::ApprovalResolved(ApprovalResolved {
1469            thread_id: pending.thread_id,
1470            turn_id: pending.turn_id,
1471            approval_id: approval_id.to_string(),
1472            tool_id: pending.tool_id,
1473            tool_name: pending.tool_name,
1474            approved,
1475            timestamp: OffsetDateTime::now_utc(),
1476        }))
1477        .await;
1478        let _ = pending.tx.send(approved);
1479        Ok(true)
1480    }
1481
1482    pub async fn request_app_server_tool_approval(
1483        &self,
1484        call: ToolCall,
1485        reason: Option<String>,
1486    ) -> anyhow::Result<bool> {
1487        let approval_id = call.id.clone();
1488        let (tx, rx) = oneshot::channel();
1489        self.pending_tool_approvals.lock().await.insert(
1490            approval_id.clone(),
1491            PendingToolApproval {
1492                thread_id: call.thread_id.clone(),
1493                turn_id: call.turn_id.clone(),
1494                tool_id: call.id.clone(),
1495                tool_name: call.name.clone(),
1496                call: call.clone(),
1497                tx,
1498            },
1499        );
1500        self.emit(RoderEvent::ApprovalRequested(ApprovalRequested {
1501            thread_id: call.thread_id.clone(),
1502            turn_id: call.turn_id.clone(),
1503            approval_id,
1504            tool_id: call.id.clone(),
1505            tool_name: call.name.clone(),
1506            reason,
1507            timestamp: OffsetDateTime::now_utc(),
1508        }))
1509        .await;
1510        Ok(rx.await.unwrap_or(false))
1511    }
1512
1513    /// Completes a pending host-executed tool call (`tools/resolve`). Returns false when the
1514    /// request id is unknown, already resolved, timed out, or cancelled by a turn interrupt.
1515    pub async fn resolve_external_tool_call(
1516        &self,
1517        request_id: &str,
1518        resolution: ExternalToolResolution,
1519    ) -> anyhow::Result<bool> {
1520        let pending = self
1521            .pending_external_tool_calls
1522            .lock()
1523            .await
1524            .remove(request_id);
1525        let Some(pending) = pending else {
1526            return Ok(false);
1527        };
1528        self.emit(RoderEvent::ExternalToolCallResolved(
1529            ExternalToolCallResolved {
1530                thread_id: pending.thread_id,
1531                turn_id: pending.turn_id,
1532                request_id: request_id.to_string(),
1533                tool_id: pending.tool_id,
1534                tool_name: pending.tool_name,
1535                outcome: ExternalToolCallOutcome::Resolved,
1536                is_error: resolution.is_error,
1537                timestamp: OffsetDateTime::now_utc(),
1538            },
1539        ))
1540        .await;
1541        let _ = pending.tx.send(resolution);
1542        Ok(true)
1543    }
1544
1545    /// Drops every pending external tool call for the turn and reports them as cancelled so an
1546    /// interrupt does not leak requests waiting on `tools/resolve`.
1547    async fn cancel_pending_external_tool_calls_for_turn(&self, turn_id: &TurnId) {
1548        let cancelled = {
1549            let mut pending = self.pending_external_tool_calls.lock().await;
1550            let request_ids = pending
1551                .iter()
1552                .filter(|(_, call)| &call.turn_id == turn_id)
1553                .map(|(request_id, _)| request_id.clone())
1554                .collect::<Vec<_>>();
1555            request_ids
1556                .into_iter()
1557                .filter_map(|request_id| pending.remove(&request_id).map(|call| (request_id, call)))
1558                .collect::<Vec<_>>()
1559        };
1560        for (request_id, call) in cancelled {
1561            self.emit(RoderEvent::ExternalToolCallResolved(
1562                ExternalToolCallResolved {
1563                    thread_id: call.thread_id,
1564                    turn_id: call.turn_id,
1565                    request_id,
1566                    tool_id: call.tool_id,
1567                    tool_name: call.tool_name,
1568                    outcome: ExternalToolCallOutcome::Cancelled,
1569                    is_error: true,
1570                    timestamp: OffsetDateTime::now_utc(),
1571                },
1572            ))
1573            .await;
1574        }
1575    }
1576
1577    pub async fn resolve_user_input(
1578        &self,
1579        request_id: &str,
1580        answers: serde_json::Value,
1581    ) -> anyhow::Result<bool> {
1582        let pending = self.pending_user_inputs.lock().await.remove(request_id);
1583        let Some(pending) = pending else {
1584            return Ok(false);
1585        };
1586        self.emit(RoderEvent::UserInputResolved(UserInputResolved {
1587            thread_id: pending.thread_id,
1588            turn_id: pending.turn_id,
1589            request_id: request_id.to_string(),
1590            answers: answers.clone(),
1591            timestamp: OffsetDateTime::now_utc(),
1592        }))
1593        .await;
1594        let _ = pending.tx.send(answers);
1595        Ok(true)
1596    }
1597
1598    async fn emit_plan_exit_resolved(
1599        &self,
1600        current: &PendingPlanExit,
1601        approved: bool,
1602        resolved_mode: PolicyMode,
1603    ) {
1604        self.emit(RoderEvent::PolicyExitPlanResolved(PolicyExitPlanResolved {
1605            thread_id: current.thread_id.clone(),
1606            turn_id: current.turn_id.clone(),
1607            request_id: current.request_id.clone(),
1608            approved,
1609            target_mode: current.target_mode,
1610            resolved_mode,
1611            timestamp: OffsetDateTime::now_utc(),
1612        }))
1613        .await;
1614    }
1615
1616    pub async fn select_provider(
1617        &self,
1618        provider: String,
1619        model: Option<String>,
1620        reasoning: Option<String>,
1621    ) -> anyhow::Result<RuntimeConfig> {
1622        let next = self
1623            .preview_provider_selection(provider, model, reasoning)
1624            .await?;
1625        let mut cfg = self.config.write().await;
1626        *cfg = next;
1627        Ok(cfg.clone())
1628    }
1629
1630    pub async fn preview_provider_selection(
1631        &self,
1632        provider: String,
1633        model: Option<String>,
1634        reasoning: Option<String>,
1635    ) -> anyhow::Result<RuntimeConfig> {
1636        self.engine_for(&provider)?;
1637        let mut cfg = self.config.read().await.clone();
1638        cfg.default_provider = provider;
1639        if let Some(model) = model {
1640            cfg.default_model = model;
1641        }
1642        if let Some(reasoning) = reasoning {
1643            if reasoning == REASONING_NONE
1644                && !model_supports_reasoning(&cfg.default_model, &reasoning)
1645            {
1646                return Ok(cfg.clone());
1647            }
1648            validate_reasoning_effort(&cfg.default_model, &reasoning)?;
1649            cfg.reasoning = Some(reasoning);
1650        }
1651        Ok(cfg)
1652    }
1653
1654    pub async fn effective_reasoning(&self) -> String {
1655        let cfg = self.config.read().await;
1656        effective_reasoning_for_model(&cfg, &cfg.default_model)
1657    }
1658
1659    pub fn effective_reasoning_for_config(cfg: &RuntimeConfig) -> String {
1660        effective_reasoning_for_model(cfg, &cfg.default_model)
1661    }
1662
1663    pub async fn set_dynamic_workflow_effort(
1664        &self,
1665        effort_profile: DynamicWorkflowEffortProfile,
1666    ) -> RuntimeConfig {
1667        let mut cfg = self.config.write().await;
1668        cfg.dynamic_workflows.effort_profile = effort_profile;
1669        cfg.clone()
1670    }
1671
1672    pub async fn dynamic_workflow_trigger_decision(
1673        &self,
1674        message: &str,
1675    ) -> WorkflowTriggerDecision {
1676        let cfg = self.config.read().await;
1677        classify_workflow_trigger(message, &cfg.dynamic_workflows)
1678    }
1679
1680    pub async fn create_thread(&self, title: Option<String>) -> anyhow::Result<ThreadMetadata> {
1681        self.create_thread_with(CreateThreadRequest {
1682            title,
1683            workspace: self.workspace.display().to_string(),
1684            workspace_id: None,
1685            root_id: None,
1686            provider: None,
1687            model: None,
1688            selection_mode: None,
1689            tool_allowlist: Vec::new(),
1690            developer_instructions: None,
1691            external_tools: Vec::new(),
1692            runner: None,
1693        })
1694        .await
1695    }
1696
1697    /**
1698     * Resolves an explicit thread-runner selection into a persisted binding.
1699     * Fails fast at thread creation when the provider is missing or rejects
1700     * the destination, instead of surfacing the error on the first tool call.
1701     */
1702    async fn resolve_thread_runner_binding(
1703        &self,
1704        thread_id: &str,
1705        selection: ThreadRunnerSelection,
1706    ) -> anyhow::Result<ThreadRunnerBinding> {
1707        let provider = self
1708            .registry
1709            .remote_runner_providers
1710            .iter()
1711            .find(|provider| provider.id() == selection.provider_id)
1712            .cloned()
1713            .ok_or_else(|| {
1714                anyhow::anyhow!(
1715                    "remote runner provider {:?} is not installed",
1716                    selection.provider_id
1717                )
1718            })?;
1719        let workspace = selection.workspace.trim();
1720        anyhow::ensure!(
1721            std::path::Path::new(workspace).is_absolute(),
1722            "runner workspace must be an absolute path on the runner: {workspace:?}"
1723        );
1724        let mut read_roots = Vec::with_capacity(selection.read_roots.len());
1725        for read_root in &selection.read_roots {
1726            let trimmed = read_root.trim();
1727            anyhow::ensure!(
1728                std::path::Path::new(trimmed).is_absolute(),
1729                "runner read root must be an absolute path on the runner: {trimmed:?}"
1730            );
1731            read_roots.push(PathBuf::from(trimmed));
1732        }
1733        let destination = RunnerDestination {
1734            id: format!("thread-{thread_id}"),
1735            provider_id: selection.provider_id,
1736            config: selection.config,
1737            default_manifest: roder_api::remote_runner::RunnerManifest::default(),
1738        };
1739        provider.validate_destination(&destination).await?;
1740        Ok(ThreadRunnerBinding {
1741            destination,
1742            workspace: PathBuf::from(workspace),
1743            read_roots,
1744        })
1745    }
1746
1747    /**
1748     * Build a per-thread binding from a runtime-level destination (selected via
1749     * the TUI runner picker or config `default_destination`) when the
1750     * destination's provider advertises a default workspace. Returns `None` when
1751     * there is no destination or the provider opts out (no default workspace),
1752     * preserving the legacy behavior where such threads keep local tools.
1753     */
1754    async fn synthesize_runtime_runner_binding(
1755        &self,
1756        thread_id: &str,
1757        destination: Option<RunnerDestination>,
1758    ) -> anyhow::Result<Option<ThreadRunnerBinding>> {
1759        let Some(destination) = destination else {
1760            return Ok(None);
1761        };
1762        let Some(provider) = self
1763            .registry
1764            .remote_runner_providers
1765            .iter()
1766            .find(|provider| provider.id() == destination.provider_id)
1767        else {
1768            return Ok(None);
1769        };
1770        // An explicit workspace in the destination config wins over the
1771        // provider default; absence of both means the provider opts out.
1772        let workspace = destination
1773            .config
1774            .get("working_dir")
1775            .and_then(serde_json::Value::as_str)
1776            .map(str::to_string)
1777            .or_else(|| provider.default_workspace());
1778        let Some(workspace) = workspace else {
1779            return Ok(None);
1780        };
1781        let selection = ThreadRunnerSelection {
1782            provider_id: destination.provider_id.clone(),
1783            config: destination.config.clone(),
1784            workspace,
1785            read_roots: Vec::new(),
1786        };
1787        Ok(Some(
1788            self.resolve_thread_runner_binding(thread_id, selection)
1789                .await?,
1790        ))
1791    }
1792
1793    /// Validates a runner selection without creating a thread; the placeholder destination id only appears in error messages.
1794    pub async fn validate_thread_runner_selection(
1795        &self,
1796        selection: ThreadRunnerSelection,
1797    ) -> anyhow::Result<()> {
1798        self.resolve_thread_runner_binding("validate", selection)
1799            .await
1800            .map(|_| ())
1801    }
1802
1803    pub async fn create_thread_with(
1804        &self,
1805        req: CreateThreadRequest,
1806    ) -> anyhow::Result<ThreadMetadata> {
1807        tool_advertisement::validate_external_tool_names(&req.external_tools)?;
1808        let cfg = self.config.read().await.clone();
1809        let now = OffsetDateTime::now_utc();
1810        let workspace = validate_thread_workspace(&req.workspace)?;
1811        let provider = req.provider.unwrap_or(cfg.default_provider);
1812        let model = req.model.unwrap_or(cfg.default_model);
1813        let selection_mode = req
1814            .selection_mode
1815            .unwrap_or_else(|| ModelSelectionMode::manual(provider.clone(), model.clone(), None));
1816        let thread_id = uuid::Uuid::new_v4().to_string();
1817        let runner_binding = match req.runner {
1818            Some(selection) => Some(
1819                self.resolve_thread_runner_binding(&thread_id, selection)
1820                    .await?,
1821            ),
1822            // No explicit per-thread selection: if a runtime-level destination
1823            // is active and its provider advertises a default workspace, bind
1824            // the new thread so its coding tools route into the runner. This is
1825            // what makes a TUI/config-selected runner (e.g. Blaxel) actually
1826            // execute tools in the sandbox instead of locally. Providers that
1827            // return no default workspace stay unbound (legacy local-tools).
1828            None => {
1829                self.synthesize_runtime_runner_binding(
1830                    &thread_id,
1831                    cfg.remote_runner_destination.clone(),
1832                )
1833                .await?
1834            }
1835        };
1836        let runner_destination = runner_binding
1837            .as_ref()
1838            .map(|binding| binding.destination.clone())
1839            .or_else(|| cfg.remote_runner_destination.clone());
1840        let metadata = ThreadMetadata {
1841            thread_id,
1842            title: req.title,
1843            workspace,
1844            workspace_id: req.workspace_id,
1845            root_id: req.root_id,
1846            provider: Some(provider),
1847            model: Some(model),
1848            selection_mode: Some(selection_mode),
1849            tool_allowlist: req.tool_allowlist,
1850            developer_instructions: req.developer_instructions,
1851            external_tools: req.external_tools,
1852            runner_destination,
1853            runner_state: None,
1854            runner_binding,
1855            created_at: now,
1856            updated_at: now,
1857            message_count: 0,
1858            usage: None,
1859            parent_thread_id: None,
1860            forked_from_turn_id: None,
1861            workspace_fork: None,
1862        };
1863
1864        let metadata = if let Some(store) = &self.thread_store {
1865            store.create_thread(metadata).await?
1866        } else {
1867            metadata
1868        };
1869        self.emit(RoderEvent::ThreadCreated(ThreadCreated {
1870            thread_id: metadata.thread_id.clone(),
1871            timestamp: OffsetDateTime::now_utc(),
1872        }))
1873        .await;
1874        Ok(metadata)
1875    }
1876
1877    pub async fn list_threads(&self) -> anyhow::Result<Vec<ThreadMetadata>> {
1878        if let Some(store) = &self.thread_store {
1879            return store.list_threads().await;
1880        }
1881        Ok(Vec::new())
1882    }
1883
1884    pub async fn list_threads_page(
1885        &self,
1886        options: roder_api::thread::ThreadListOptions,
1887    ) -> anyhow::Result<roder_api::thread::ThreadListPage> {
1888        if let Some(store) = &self.thread_store {
1889            return store.list_threads_page(options).await;
1890        }
1891        Ok(roder_api::thread::ThreadListPage::default())
1892    }
1893
1894    pub async fn load_thread_metadata(
1895        &self,
1896        thread_id: &str,
1897    ) -> anyhow::Result<Option<roder_api::thread::ThreadMetadata>> {
1898        if let Some(store) = &self.thread_store {
1899            return store.load_thread_metadata(&thread_id.to_string()).await;
1900        }
1901        Ok(None)
1902    }
1903
1904    pub async fn archive_thread(&self, thread_id: &str) -> anyhow::Result<bool> {
1905        let archived = if let Some(store) = &self.thread_store {
1906            store.archive_thread(&thread_id.to_string()).await?
1907        } else {
1908            false
1909        };
1910        if archived {
1911            let workspace = self.config.read().await.workspace.clone();
1912            crate::hooks::run_lifecycle(
1913                self,
1914                &thread_id.to_string(),
1915                &SESSION_HOOK_TURN_ID.to_string(),
1916                workspace.as_deref(),
1917                "SessionEnd",
1918                Some("clear"),
1919                serde_json::json!({"reason": "archive"}),
1920            )
1921            .await;
1922            self.thread_item_cache
1923                .lock()
1924                .await
1925                .remove_thread(&thread_id.to_string());
1926            self.evict_runner_session(&thread_id.to_string()).await;
1927        }
1928        Ok(archived)
1929    }
1930
1931    pub async fn start_team(&self, req: TeamStartRequest) -> anyhow::Result<TeamState> {
1932        let cfg = self.config.read().await.clone();
1933        let workspace = self.workspace.display().to_string();
1934        let lead_thread_id = match req.lead_thread_id {
1935            Some(thread_id) => thread_id,
1936            None => {
1937                self.create_thread_with(CreateThreadRequest {
1938                    title: Some("Team lead".to_string()),
1939                    workspace: workspace.clone(),
1940                    workspace_id: None,
1941                    root_id: None,
1942                    provider: None,
1943                    model: None,
1944                    selection_mode: None,
1945                    tool_allowlist: Vec::new(),
1946                    developer_instructions: None,
1947                    external_tools: Vec::new(),
1948                    runner: None,
1949                })
1950                .await?
1951                .thread_id
1952            }
1953        };
1954        let team_id = uuid::Uuid::new_v4().to_string();
1955        let active_lead_turn_id = self.active_turn_for_thread(&lead_thread_id).await;
1956        let lead_selection = match active_lead_turn_id.as_ref() {
1957            Some(turn_id) => self
1958                .active_turn_selections
1959                .read()
1960                .await
1961                .get(turn_id)
1962                .cloned(),
1963            None => self.selection_mode_for_thread(&lead_thread_id).await?,
1964        };
1965        let lead_concrete_selection = lead_selection
1966            .as_ref()
1967            .map(ModelSelectionMode::concrete_selection);
1968        let mut lead = crate::teams::lead_member(
1969            lead_thread_id.clone(),
1970            lead_concrete_selection
1971                .as_ref()
1972                .map(|selection| selection.provider.clone())
1973                .or_else(|| Some(cfg.default_provider.clone())),
1974            lead_concrete_selection
1975                .as_ref()
1976                .map(|selection| selection.model.clone())
1977                .or_else(|| Some(cfg.default_model.clone())),
1978            cfg.policy_mode,
1979        );
1980        if let Some(turn_id) = active_lead_turn_id {
1981            lead.current_turn_id = Some(turn_id);
1982            lead.status = TeamMemberStatus::Running;
1983        }
1984        let mut members = vec![lead];
1985
1986        for (index, member) in req.members.into_iter().enumerate() {
1987            let thread = self
1988                .create_thread_with(CreateThreadRequest {
1989                    title: Some(member.name.clone()),
1990                    workspace: workspace.clone(),
1991                    workspace_id: None,
1992                    root_id: None,
1993                    provider: member.model_provider.clone(),
1994                    model: member.model.clone(),
1995                    selection_mode: None,
1996                    tool_allowlist: Vec::new(),
1997                    developer_instructions: None,
1998                    external_tools: Vec::new(),
1999                    runner: None,
2000                })
2001                .await?;
2002            let member_id = format!("member-{}", index + 1);
2003            let descriptor = crate::teams::teammate_member(
2004                member_id.clone(),
2005                member.name,
2006                thread.thread_id.clone(),
2007                member.model_provider.or(thread.provider),
2008                member.model.or(thread.model),
2009                cfg.policy_mode,
2010            );
2011            members.push(descriptor);
2012        }
2013
2014        let now = OffsetDateTime::now_utc();
2015        let team = self
2016            .teams
2017            .insert(TeamState {
2018                id: team_id.clone(),
2019                lead_thread_id: lead_thread_id.clone(),
2020                display_mode: req.display_mode,
2021                members,
2022                mailbox: Vec::new(),
2023                tasks: Vec::new(),
2024                created_at: now,
2025                updated_at: now,
2026            })
2027            .await?;
2028        self.emit(RoderEvent::TeamStarted(TeamStarted {
2029            team_id: team_id.clone(),
2030            lead_thread_id,
2031            display_mode: team.display_mode,
2032            members: team.members.clone(),
2033            tasks: team.tasks.clone(),
2034            timestamp: OffsetDateTime::now_utc(),
2035        }))
2036        .await;
2037        for member in team
2038            .members
2039            .iter()
2040            .filter(|member| member.role != roder_api::teams::TeamMemberRole::Lead)
2041        {
2042            self.emit(RoderEvent::TeamMemberStarted(TeamMemberStarted {
2043                team_id: team_id.clone(),
2044                member: member.clone(),
2045                timestamp: OffsetDateTime::now_utc(),
2046            }))
2047            .await;
2048        }
2049        Ok(team)
2050    }
2051
2052    pub async fn list_teams(&self) -> Vec<TeamState> {
2053        let active_thread_ids = self
2054            .active_turns
2055            .read()
2056            .await
2057            .values()
2058            .map(|handle| handle.thread_id.clone())
2059            .collect::<std::collections::HashSet<_>>();
2060        self.teams
2061            .list()
2062            .await
2063            .into_iter()
2064            .map(|team| runtime_local_team_view(team, &active_thread_ids))
2065            .collect()
2066    }
2067
2068    pub async fn read_team(&self, team_id: &str) -> Option<TeamState> {
2069        let active_thread_ids = self
2070            .active_turns
2071            .read()
2072            .await
2073            .values()
2074            .map(|handle| handle.thread_id.clone())
2075            .collect::<std::collections::HashSet<_>>();
2076        self.teams
2077            .get(team_id)
2078            .await
2079            .map(|team| runtime_local_team_view(team, &active_thread_ids))
2080    }
2081
2082    pub async fn start_team_member(
2083        &self,
2084        team_id: &str,
2085        req: TeamMemberStartRequest,
2086    ) -> anyhow::Result<TeamState> {
2087        self.start_team_member_with_selection(team_id, req, None)
2088            .await
2089    }
2090
2091    pub async fn message_team_member(
2092        self: &Arc<Self>,
2093        team_id: &str,
2094        member_id: &str,
2095        message: String,
2096    ) -> anyhow::Result<TurnId> {
2097        let team = self
2098            .read_team(team_id)
2099            .await
2100            .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2101        self.followup_team_member(&team.lead_thread_id, team_id, member_id, message)
2102            .await
2103    }
2104
2105    /// Queue a mailbox message without starting an idle agent. If the target is
2106    /// already running, the message is delivered at the next inference boundary.
2107    pub(crate) async fn queue_team_member_message(
2108        self: &Arc<Self>,
2109        caller_thread_id: &ThreadId,
2110        team_id: &str,
2111        member_id: &str,
2112        message: String,
2113    ) -> anyhow::Result<Option<TurnId>> {
2114        let _delivery_guard = self.agent_team_spawn_lock.lock().await;
2115        let team = self
2116            .read_team(team_id)
2117            .await
2118            .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2119        let member = team
2120            .members
2121            .iter()
2122            .find(|member| member.id == member_id)
2123            .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2124            .clone();
2125        if member.status == TeamMemberStatus::Closed {
2126            anyhow::bail!("subagent {} is closed", member.name);
2127        }
2128        let from_member_id = team
2129            .members
2130            .iter()
2131            .find(|candidate| candidate.thread_id == *caller_thread_id)
2132            .map(|candidate| candidate.id.clone());
2133        self.teams
2134            .append_mailbox_message(
2135                team_id,
2136                from_member_id,
2137                member_id.to_string(),
2138                TeamMailboxMessageKind::Message,
2139                message,
2140            )
2141            .await?;
2142        let Some(turn_id) = self.active_turn_for_thread(&member.thread_id).await else {
2143            return Ok(None);
2144        };
2145        self.deliver_pending_team_mailbox(team_id, member_id, member.thread_id, turn_id.clone())
2146            .await?;
2147        Ok(Some(turn_id))
2148    }
2149
2150    /// Deliver queued messages and start an idle agent, or steer its active turn.
2151    pub(crate) async fn followup_team_member(
2152        self: &Arc<Self>,
2153        caller_thread_id: &ThreadId,
2154        team_id: &str,
2155        member_id: &str,
2156        message: String,
2157    ) -> anyhow::Result<TurnId> {
2158        let _spawn_guard = self.agent_team_spawn_lock.lock().await;
2159        self.followup_team_member_locked(caller_thread_id, team_id, member_id, message)
2160            .await
2161    }
2162
2163    async fn followup_team_member_locked(
2164        self: &Arc<Self>,
2165        caller_thread_id: &ThreadId,
2166        team_id: &str,
2167        member_id: &str,
2168        message: String,
2169    ) -> anyhow::Result<TurnId> {
2170        let team = self
2171            .read_team(team_id)
2172            .await
2173            .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2174        let member = team
2175            .members
2176            .iter()
2177            .find(|member| member.id == member_id)
2178            .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2179            .clone();
2180        if member.status == TeamMemberStatus::Closed {
2181            anyhow::bail!("subagent {} is closed", member.name);
2182        }
2183        let from_member_id = team
2184            .members
2185            .iter()
2186            .find(|candidate| candidate.thread_id == *caller_thread_id)
2187            .map(|candidate| candidate.id.clone());
2188        self.teams
2189            .append_mailbox_message(
2190                team_id,
2191                from_member_id,
2192                member_id.to_string(),
2193                TeamMailboxMessageKind::NewTask,
2194                message,
2195            )
2196            .await?;
2197        if let Some(turn_id) = self.active_turn_for_thread(&member.thread_id).await {
2198            self.deliver_pending_team_mailbox(
2199                team_id,
2200                member_id,
2201                member.thread_id,
2202                turn_id.clone(),
2203            )
2204            .await?;
2205            return Ok(turn_id);
2206        }
2207
2208        self.ensure_codex_v2_team_capacity(&team, &member.id, None)
2209            .await?;
2210        let metadata = self.load_thread_metadata(&member.thread_id).await?;
2211        let inherited_context = self
2212            .team_member_turn_contexts
2213            .lock()
2214            .await
2215            .get(&member.thread_id)
2216            .cloned();
2217        let workspace = inherited_context
2218            .as_ref()
2219            .map(|context| context.workspace.clone())
2220            .or_else(|| metadata.as_ref().map(|metadata| metadata.workspace.clone()))
2221            .unwrap_or_else(|| self.workspace.display().to_string());
2222        let member_thread_id = member.thread_id;
2223        let turn_id = self
2224            .start_turn(StartTurnRequest {
2225                thread_id: member_thread_id.clone(),
2226                message: String::new(),
2227                images: Vec::new(),
2228                provider_override: member.model_provider,
2229                model_override: member.model,
2230                reasoning_override: metadata
2231                    .as_ref()
2232                    .and_then(|metadata| metadata.selection_mode.as_ref())
2233                    .and_then(ModelSelectionMode::reasoning)
2234                    .map(str::to_string),
2235                workspace,
2236                instructions: inherited_context
2237                    .as_ref()
2238                    .map(|context| context.instructions.clone())
2239                    .unwrap_or_else(crate::default_instructions),
2240                developer_context: inherited_context
2241                    .as_ref()
2242                    .and_then(|context| context.developer_context.clone()),
2243                task_ledger_required: false,
2244                service_tier_override: None,
2245            })
2246            .await?;
2247        Ok(turn_id)
2248    }
2249
2250    pub async fn set_team_member_policy_mode(
2251        &self,
2252        team_id: &str,
2253        member_id: &str,
2254        policy_mode: PolicyMode,
2255    ) -> anyhow::Result<TeamState> {
2256        self.teams
2257            .set_member_policy_mode(team_id, member_id, policy_mode)
2258            .await
2259    }
2260
2261    pub async fn interrupt_team_member(
2262        &self,
2263        team_id: &str,
2264        member_id: &str,
2265    ) -> anyhow::Result<Option<TurnId>> {
2266        let _interrupt_guard = self.agent_team_spawn_lock.lock().await;
2267        let team = self
2268            .read_team(team_id)
2269            .await
2270            .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2271        let member = team
2272            .members
2273            .iter()
2274            .find(|member| member.id == member_id)
2275            .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2276            .clone();
2277        if member.status != TeamMemberStatus::Running {
2278            return Ok(None);
2279        }
2280        let Some(turn_id) = member.current_turn_id.clone() else {
2281            return Ok(None);
2282        };
2283        if self
2284            .active_turn_for_thread(&member.thread_id)
2285            .await
2286            .as_ref()
2287            != Some(&turn_id)
2288        {
2289            return Ok(None);
2290        }
2291        self.interrupt_turn(member.thread_id.clone(), turn_id.clone())
2292            .await?;
2293        // Make queued mailbox work immediately eligible for the replacement turn while the
2294        // lifecycle lock still prevents a follow-up from starting. Delivery acknowledgements
2295        // are turn-owned, so a late acknowledgement from the aborted turn cannot steal a
2296        // reservation acquired by its replacement.
2297        self.teams
2298            .release_mailbox_reservations_for_turn(&turn_id)
2299            .await;
2300        self.teams
2301            .update_member(team_id, member_id, |member| {
2302                member.status = TeamMemberStatus::Interrupted;
2303                member.current_turn_id = None;
2304                member.terminal_error = None;
2305            })
2306            .await?;
2307        self.emit(RoderEvent::TeamMemberCompleted(TeamMemberCompleted {
2308            team_id: team_id.to_string(),
2309            member_id: member_id.to_string(),
2310            member_thread_id: member.thread_id,
2311            turn_id: Some(turn_id.clone()),
2312            status: TeamMemberStatus::Interrupted,
2313            final_message: member.final_message,
2314            error: None,
2315            timestamp: OffsetDateTime::now_utc(),
2316        }))
2317        .await;
2318        Ok(Some(turn_id))
2319    }
2320
2321    pub async fn close_team_member(
2322        &self,
2323        team_id: &str,
2324        member_id: &str,
2325    ) -> anyhow::Result<roder_api::teams::TeamMemberDescriptor> {
2326        let team = self
2327            .read_team(team_id)
2328            .await
2329            .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2330        let member = team
2331            .members
2332            .iter()
2333            .find(|member| member.id == member_id)
2334            .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2335            .clone();
2336        if member.role == roder_api::teams::TeamMemberRole::Lead {
2337            anyhow::bail!("team lead cannot be closed as a subagent");
2338        }
2339        let interrupted_turn_id = if member.status == TeamMemberStatus::Running {
2340            if let Some(turn_id) = member.current_turn_id.clone() {
2341                self.interrupt_turn(member.thread_id.clone(), turn_id.clone())
2342                    .await?;
2343                Some(turn_id)
2344            } else {
2345                None
2346            }
2347        } else {
2348            member.current_turn_id.clone()
2349        };
2350        let updated = self
2351            .teams
2352            .update_member(team_id, member_id, |member| {
2353                member.status = TeamMemberStatus::Closed;
2354                member.current_turn_id = None;
2355            })
2356            .await?;
2357        let closed = updated
2358            .members
2359            .iter()
2360            .find(|member| member.id == member_id)
2361            .cloned()
2362            .ok_or_else(|| anyhow::anyhow!("closed team member disappeared"))?;
2363        self.team_member_turn_contexts
2364            .lock()
2365            .await
2366            .remove(&closed.thread_id);
2367        self.emit(RoderEvent::TeamMemberCompleted(TeamMemberCompleted {
2368            team_id: team_id.to_string(),
2369            member_id: closed.id.clone(),
2370            member_thread_id: closed.thread_id.clone(),
2371            turn_id: interrupted_turn_id,
2372            status: TeamMemberStatus::Closed,
2373            final_message: closed.final_message.clone(),
2374            error: closed.terminal_error.clone(),
2375            timestamp: OffsetDateTime::now_utc(),
2376        }))
2377        .await;
2378        Ok(closed)
2379    }
2380
2381    pub async fn cleanup_team(&self, team_id: &str, force: bool) -> anyhow::Result<bool> {
2382        let Some(team) = self.read_team(team_id).await else {
2383            return Ok(false);
2384        };
2385        if !force
2386            && team
2387                .members
2388                .iter()
2389                .any(|member| member.status == TeamMemberStatus::Running)
2390        {
2391            anyhow::bail!("team {team_id:?} has active teammates; use forced cleanup");
2392        }
2393        if force {
2394            for member in team.members.iter().filter(|member| {
2395                member.role != roder_api::teams::TeamMemberRole::Lead
2396                    && member.status == TeamMemberStatus::Running
2397            }) {
2398                if let Some(turn_id) = member
2399                    .current_turn_id
2400                    .clone()
2401                    .or(self.active_turn_for_thread(&member.thread_id).await)
2402                {
2403                    let _ = self.interrupt_turn(member.thread_id.clone(), turn_id).await;
2404                }
2405            }
2406        }
2407        let removed = self.teams.remove(team_id).await?.is_some();
2408        if removed {
2409            let member_thread_ids = team
2410                .members
2411                .iter()
2412                .map(|member| member.thread_id.clone())
2413                .collect::<std::collections::HashSet<_>>();
2414            self.team_member_turn_contexts
2415                .lock()
2416                .await
2417                .retain(|thread_id, _| !member_thread_ids.contains(thread_id));
2418            self.emit(RoderEvent::TeamCleanupCompleted(TeamCleanupCompleted {
2419                team_id: team_id.to_string(),
2420                forced: force,
2421                timestamp: OffsetDateTime::now_utc(),
2422            }))
2423            .await;
2424        }
2425        Ok(removed)
2426    }
2427
2428    pub async fn effective_policy_mode_for_thread(&self, thread_id: &str) -> PolicyMode {
2429        if let Some(mode) = self.teams.policy_mode_for_thread(thread_id).await {
2430            return mode;
2431        }
2432        self.status().await.policy_mode
2433    }
2434
2435    async fn complete_team_member_turn_with_result(
2436        &self,
2437        thread_id: &ThreadId,
2438        turn_id: &TurnId,
2439        status: TeamMemberStatus,
2440        final_message: Option<String>,
2441        terminal_error: Option<String>,
2442    ) -> anyhow::Result<()> {
2443        let _delivery_guard = self.agent_team_spawn_lock.lock().await;
2444        let Some((team_id, member)) = self
2445            .teams
2446            .complete_member_turn(
2447                thread_id,
2448                turn_id,
2449                status,
2450                final_message.clone(),
2451                terminal_error.clone(),
2452            )
2453            .await?
2454        else {
2455            return Ok(());
2456        };
2457        if let Some(parent_thread_id) = member.parent_thread_id.as_ref()
2458            && let Some(team) = self.read_team(&team_id).await
2459            && let Some(parent) = team
2460                .members
2461                .iter()
2462                .find(|candidate| candidate.thread_id == *parent_thread_id)
2463        {
2464            let identity = member
2465                .agent_path
2466                .as_deref()
2467                .or(member.task_name.as_deref())
2468                .unwrap_or(&member.name);
2469            let mut report = format!("Agent {identity} finished with status {status:?}.");
2470            if let Some(message) = final_message.as_deref()
2471                && !message.trim().is_empty()
2472            {
2473                report.push_str("\n\nFinal result:\n");
2474                report.push_str(message);
2475            }
2476            if let Some(error) = terminal_error.as_deref()
2477                && !error.trim().is_empty()
2478            {
2479                report.push_str("\n\nTerminal error:\n");
2480                report.push_str(error);
2481            }
2482            let mailbox_appended = self
2483                .teams
2484                .append_mailbox_message(
2485                    &team_id,
2486                    Some(member.id.clone()),
2487                    parent.id.clone(),
2488                    TeamMailboxMessageKind::FinalAnswer,
2489                    report,
2490                )
2491                .await
2492                .is_ok();
2493            if mailbox_appended
2494                && let Some(parent_turn_id) = self.active_turn_for_thread(parent_thread_id).await
2495            {
2496                // The parent may finish between the active-turn lookup and enqueue. Its durable
2497                // mailbox entry remains pending for the next turn, while terminal observers must
2498                // still receive TeamMemberCompleted for this child.
2499                let _ = self
2500                    .deliver_pending_team_mailbox(
2501                        &team_id,
2502                        &parent.id,
2503                        parent_thread_id.clone(),
2504                        parent_turn_id,
2505                    )
2506                    .await;
2507            }
2508        }
2509        self.emit(RoderEvent::TeamMemberCompleted(TeamMemberCompleted {
2510            team_id,
2511            member_id: member.id,
2512            member_thread_id: member.thread_id,
2513            turn_id: Some(turn_id.clone()),
2514            status,
2515            final_message,
2516            error: terminal_error,
2517            timestamp: OffsetDateTime::now_utc(),
2518        }))
2519        .await;
2520        Ok(())
2521    }
2522
2523    /// Stops accepting new turns, requests interruption for every active turn,
2524    /// and waits for their runtime tasks to reach terminal cleanup up to
2525    /// `timeout`. Repeated calls are safe and continue draining the same set.
2526    pub async fn drain_active_turns(&self, timeout: std::time::Duration) -> RuntimeDrainOutcome {
2527        let started_at = tokio::time::Instant::now();
2528        let persistence_failures_before =
2529            self.lifecycle_persistence_failures.load(Ordering::Acquire);
2530        let turn_admission = self.turn_admission.lock().await;
2531        self.accepting_turns.store(false, Ordering::Release);
2532        let active = self
2533            .active_turns
2534            .read()
2535            .await
2536            .iter()
2537            .map(|(turn_id, handle)| (turn_id.clone(), handle.thread_id.clone()))
2538            .collect::<Vec<_>>();
2539        let draining = self
2540            .turn_drains
2541            .read()
2542            .await
2543            .iter()
2544            .map(|(turn_id, drain)| (turn_id.clone(), drain.thread_id.clone()))
2545            .collect::<Vec<_>>();
2546        drop(turn_admission);
2547
2548        let mut interrupted_turns = std::collections::BTreeMap::new();
2549        for (turn_id, thread_id) in active.iter().chain(draining.iter()) {
2550            interrupted_turns.insert(turn_id.clone(), thread_id.clone());
2551        }
2552        let interrupted_turn_ids = interrupted_turns.keys().cloned().collect::<Vec<_>>();
2553        for (turn_id, thread_id) in active {
2554            let _ = self
2555                .interrupt_turn_with_reason(thread_id, turn_id, TurnLifecycleReason::Shutdown)
2556                .await;
2557        }
2558
2559        let wait_for_empty = async {
2560            loop {
2561                let notified = self.active_turns_changed.notified();
2562                if self.active_turns.read().await.is_empty()
2563                    && self.turn_drains.read().await.is_empty()
2564                {
2565                    return;
2566                }
2567                notified.await;
2568            }
2569        };
2570        let outcome = if tokio::time::timeout(timeout, wait_for_empty).await.is_ok() {
2571            if self.lifecycle_persistence_failures.load(Ordering::Acquire)
2572                > persistence_failures_before
2573            {
2574                RuntimeDrainOutcome::PersistenceFailed {
2575                    interrupted_turn_ids,
2576                    remaining_turn_ids: Vec::new(),
2577                }
2578            } else {
2579                RuntimeDrainOutcome::Clean {
2580                    interrupted_turn_ids,
2581                }
2582            }
2583        } else {
2584            let active_remaining = self
2585                .active_turns
2586                .read()
2587                .await
2588                .iter()
2589                .map(|(turn_id, handle)| (turn_id.clone(), handle.drain.clone()))
2590                .collect::<Vec<_>>();
2591            let draining_remaining = self
2592                .turn_drains
2593                .read()
2594                .await
2595                .iter()
2596                .map(|(turn_id, drain)| (turn_id.clone(), drain.clone()))
2597                .collect::<Vec<_>>();
2598            let remaining = active_remaining
2599                .into_iter()
2600                .chain(draining_remaining)
2601                .collect::<std::collections::BTreeMap<_, _>>();
2602            let remaining_turn_ids = remaining.keys().cloned().collect::<Vec<_>>();
2603            for turn_id in &remaining_turn_ids {
2604                let handle = remaining
2605                    .get(turn_id)
2606                    .expect("remaining turn ID must have a drain handle");
2607                self.record_turn_lifecycle(
2608                    handle.thread_id.clone(),
2609                    turn_id.clone(),
2610                    TurnLifecycleState::InterruptRequested,
2611                    TurnCleanupState::TimedOut,
2612                    Some(TurnLifecycleReason::Shutdown),
2613                )
2614                .await;
2615            }
2616            if self.lifecycle_persistence_failures.load(Ordering::Acquire)
2617                > persistence_failures_before
2618            {
2619                RuntimeDrainOutcome::PersistenceFailed {
2620                    interrupted_turn_ids,
2621                    remaining_turn_ids,
2622                }
2623            } else {
2624                RuntimeDrainOutcome::DeadlineExceeded {
2625                    interrupted_turn_ids,
2626                    remaining_turn_ids,
2627                }
2628            }
2629        };
2630        self.record_lifecycle_drain_outcome(started_at, &outcome);
2631        outcome
2632    }
2633
2634    /// Enables turn submission after a prior drain attempt. This is intended
2635    /// for embedders that keep a runtime alive after a bounded drain timeout.
2636    pub fn resume_accepting_turns(&self) {
2637        self.accepting_turns.store(true, Ordering::Release);
2638    }
2639
2640    pub async fn turn_lifecycle_snapshot(
2641        &self,
2642        thread_id: &ThreadId,
2643    ) -> anyhow::Result<TurnLifecycleSnapshot> {
2644        let Some(store) = &self.thread_store else {
2645            return Ok(TurnLifecycleSnapshot::default());
2646        };
2647        let extension_states = store.load_extension_states(thread_id).await?;
2648        Ok(turn_lifecycle_snapshot(&extension_states))
2649    }
2650
2651    pub async fn load_thread(
2652        &self,
2653        thread_id: &ThreadId,
2654    ) -> anyhow::Result<Option<ThreadSnapshot>> {
2655        let loaded = if let Some(store) = &self.thread_store {
2656            store.load_thread(thread_id).await?
2657        } else {
2658            None
2659        };
2660        if let Some(snapshot) = loaded.as_ref() {
2661            let live_turn_ids = self
2662                .active_turns
2663                .read()
2664                .await
2665                .keys()
2666                .cloned()
2667                .chain(self.turn_drains.read().await.keys().cloned())
2668                .collect::<std::collections::HashSet<_>>();
2669            let lifecycle = turn_lifecycle_snapshot(&snapshot.extension_states);
2670            for record in lifecycle.records.into_iter().filter(|record| {
2671                record.state.requires_recovery() && !live_turn_ids.contains(&record.turn_id)
2672            }) {
2673                self.lifecycle_restart_reconciliations
2674                    .fetch_add(1, Ordering::AcqRel);
2675                self.record_turn_lifecycle(
2676                    record.thread_id,
2677                    record.turn_id,
2678                    TurnLifecycleState::RecoveryNeeded,
2679                    TurnCleanupState::Unknown,
2680                    Some(TurnLifecycleReason::RuntimeRestart),
2681                )
2682                .await;
2683            }
2684            self.emit(RoderEvent::ThreadLoaded(ThreadLoaded {
2685                thread_id: thread_id.clone(),
2686                timestamp: OffsetDateTime::now_utc(),
2687            }))
2688            .await;
2689        }
2690        Ok(loaded)
2691    }
2692
2693    pub async fn workspace_for_thread(&self, thread_id: &ThreadId) -> anyhow::Result<String> {
2694        if let Some(store) = &self.thread_store {
2695            let snapshot = store
2696                .load_thread(thread_id)
2697                .await?
2698                .ok_or_else(|| anyhow::anyhow!("thread not found: {thread_id}"));
2699            match snapshot {
2700                Ok(snapshot) => {
2701                    if let Some(metadata) = snapshot.metadata {
2702                        // Fork-backed threads fail closed before any write
2703                        // when their workspace was removed out-of-band.
2704                        if let Some(fork) = &metadata.workspace_fork
2705                            && fork.status == roder_api::forks::ForkStatus::Active
2706                            && !std::path::Path::new(&metadata.workspace).is_dir()
2707                        {
2708                            anyhow::bail!(
2709                                "workspace fork {} is missing its workspace at {}; restore it or \
2710                                 remove the fork before running turns in this thread",
2711                                fork.id,
2712                                metadata.workspace
2713                            );
2714                        }
2715                        return Ok(metadata.workspace);
2716                    }
2717                    eprintln!(
2718                        "thread metadata missing while resolving workspace for {thread_id}; falling back to runtime workspace"
2719                    );
2720                }
2721                Err(err) => {
2722                    eprintln!(
2723                        "thread missing while resolving workspace for {thread_id}: {err}; falling back to runtime workspace"
2724                    );
2725                }
2726            }
2727        }
2728        Ok(self.workspace.display().to_string())
2729    }
2730
2731    pub(crate) async fn selection_mode_for_thread(
2732        &self,
2733        thread_id: &ThreadId,
2734    ) -> anyhow::Result<Option<ModelSelectionMode>> {
2735        let Some(store) = &self.thread_store else {
2736            return Ok(None);
2737        };
2738        Ok(store
2739            .load_thread(thread_id)
2740            .await?
2741            .and_then(|snapshot| snapshot.metadata)
2742            .and_then(|metadata| {
2743                metadata
2744                    .selection_mode
2745                    .or_else(|| match (metadata.provider, metadata.model) {
2746                        (Some(provider), Some(model)) => {
2747                            Some(ModelSelectionMode::manual(provider, model, None))
2748                        }
2749                        _ => None,
2750                    })
2751            }))
2752    }
2753
2754    /// Concrete provider/model for configured-role subagents (`task`,
2755    /// `agent_swarm`). Prefers the live parent turn selection so children stay
2756    /// on SuperGrok/Grok (or whatever the lead is using) instead of process
2757    /// startup defaults such as openai/gpt-5.5.
2758    pub(crate) async fn parent_model_selection_for_subagents(
2759        &self,
2760        thread_id: &ThreadId,
2761        turn_id: &TurnId,
2762    ) -> Option<roder_api::inference::ModelSelection> {
2763        if let Some(selection) = self
2764            .active_turn_selections
2765            .read()
2766            .await
2767            .get(turn_id)
2768            .cloned()
2769        {
2770            return Some(selection.concrete_selection());
2771        }
2772        if let Ok(Some(selection)) = self.selection_mode_for_thread(thread_id).await {
2773            return Some(selection.concrete_selection());
2774        }
2775        let cfg = self.status().await;
2776        if cfg.default_provider.trim().is_empty() || cfg.default_model.trim().is_empty() {
2777            return None;
2778        }
2779        Some(roder_api::inference::ModelSelection {
2780            provider: cfg.default_provider,
2781            model: cfg.default_model,
2782        })
2783    }
2784
2785    /// Per-thread tool allowlist, developer instructions, and external tools persisted at thread creation.
2786    pub(crate) async fn thread_turn_overrides(
2787        &self,
2788        thread_id: &ThreadId,
2789    ) -> anyhow::Result<ThreadTurnOverrides> {
2790        let Some(store) = &self.thread_store else {
2791            return Ok(ThreadTurnOverrides::default());
2792        };
2793        Ok(store
2794            .load_thread_metadata(thread_id)
2795            .await?
2796            .map(|metadata| ThreadTurnOverrides {
2797                tool_allowlist: metadata.tool_allowlist,
2798                developer_instructions: metadata.developer_instructions,
2799                external_tools: metadata.external_tools,
2800            })
2801            .unwrap_or_default())
2802    }
2803
2804    pub async fn set_thread_selection_mode(
2805        &self,
2806        thread_id: &ThreadId,
2807        selection_mode: ModelSelectionMode,
2808    ) -> anyhow::Result<()> {
2809        let Some(store) = &self.thread_store else {
2810            return Ok(());
2811        };
2812        let Some(snapshot) = store.load_thread(thread_id).await? else {
2813            anyhow::bail!("thread not found: {thread_id}");
2814        };
2815        let Some(mut metadata) = snapshot.metadata else {
2816            return Ok(());
2817        };
2818        let concrete = selection_mode.concrete_selection();
2819        metadata.provider = Some(concrete.provider);
2820        metadata.model = Some(concrete.model);
2821        metadata.selection_mode = Some(selection_mode);
2822        metadata.updated_at = OffsetDateTime::now_utc();
2823        store.update_thread_metadata(metadata).await?;
2824        Ok(())
2825    }
2826
2827    async fn runner_session_for_thread(
2828        &self,
2829        thread_id: &ThreadId,
2830    ) -> anyhow::Result<Option<(RunnerDestination, Arc<dyn RemoteRunnerSession>)>> {
2831        let metadata = if let Some(store) = &self.thread_store {
2832            store.load_thread_metadata(thread_id).await?
2833        } else {
2834            None
2835        };
2836        // An explicit per-thread binding wins over the runtime-level destination.
2837        let destination = metadata
2838            .as_ref()
2839            .and_then(|metadata| metadata.runner_binding.as_ref())
2840            .map(|binding| binding.destination.clone())
2841            .or(self.config.read().await.remote_runner_destination.clone());
2842        let Some(destination) = destination else {
2843            self.evict_runner_session(thread_id).await;
2844            return Ok(None);
2845        };
2846        let provider = self
2847            .registry
2848            .remote_runner_providers
2849            .iter()
2850            .find(|provider| provider.id() == destination.provider_id)
2851            .cloned()
2852            .ok_or_else(|| {
2853                anyhow::anyhow!(
2854                    "remote runner provider {:?} is not installed",
2855                    destination.provider_id
2856                )
2857            })?;
2858        let persisted_state = metadata.and_then(|metadata| metadata.runner_state);
2859        let slot = {
2860            let mut sessions = self.runner_sessions.lock().await;
2861            match sessions.get(thread_id) {
2862                Some(slot) if slot.matches(&destination) => slot.clone(),
2863                _ => {
2864                    let slot = Arc::new(RunnerSessionSlot::empty(&destination));
2865                    sessions.insert(thread_id.clone(), slot.clone());
2866                    slot
2867                }
2868            }
2869        };
2870
2871        loop {
2872            let initialized = slot.initialized.notified();
2873            let mut state = slot.state.lock().await;
2874            match &*state {
2875                RunnerSessionSlotState::Ready(cached) => {
2876                    return Ok(Some((cached.destination.clone(), cached.session.clone())));
2877                }
2878                RunnerSessionSlotState::Failed(error) => {
2879                    return Err(anyhow::anyhow!(error.to_string()));
2880                }
2881                RunnerSessionSlotState::Initializing => {
2882                    drop(state);
2883                    initialized.await;
2884                }
2885                RunnerSessionSlotState::Empty => {
2886                    *state = RunnerSessionSlotState::Initializing;
2887                    drop(state);
2888                    self.spawn_runner_session_initialization(
2889                        thread_id.clone(),
2890                        destination.clone(),
2891                        provider.clone(),
2892                        persisted_state.clone(),
2893                        slot.clone(),
2894                    );
2895                }
2896            }
2897        }
2898    }
2899
2900    fn spawn_runner_session_initialization(
2901        &self,
2902        thread_id: ThreadId,
2903        destination: RunnerDestination,
2904        provider: Arc<dyn RemoteRunnerProvider>,
2905        persisted_state: Option<RunnerSessionState>,
2906        slot: Arc<RunnerSessionSlot>,
2907    ) {
2908        let thread_store = self.thread_store.clone();
2909        let runner_sessions = self.runner_sessions.clone();
2910        // The task intentionally outlives the turn that first requested the
2911        // workspace. Interrupting that turn must not strand the slot in
2912        // `Initializing` and deadlock every later tool or lifecycle call.
2913        drop(tokio::spawn(async move {
2914            let initialized = async {
2915                let session = if let Some(state) = persisted_state
2916                    && state.provider_id == destination.provider_id
2917                    && state.destination_id == destination.id
2918                {
2919                    match provider.resume_session(state).await {
2920                        Ok(session) => session,
2921                        Err(_) => provider.create_session(destination.clone()).await?,
2922                    }
2923                } else {
2924                    provider.create_session(destination.clone()).await?
2925                };
2926                let resolved = (destination.clone(), session.clone());
2927                // Persist before releasing the singleflight or dispatching
2928                // the first tool. A later process can rejoin even if the turn
2929                // that requested initialization has already been interrupted.
2930                Self::persist_runner_state_in_store(
2931                    thread_store.as_ref(),
2932                    &thread_id,
2933                    Some(&resolved),
2934                )
2935                .await?;
2936                Ok::<_, anyhow::Error>(CachedRunnerSession {
2937                    destination,
2938                    session,
2939                })
2940            }
2941            .await;
2942
2943            match initialized {
2944                Ok(cached) => {
2945                    *slot.state.lock().await = RunnerSessionSlotState::Ready(cached);
2946                    slot.initialized.notify_waiters();
2947                }
2948                Err(error) => {
2949                    let message: Arc<str> = Arc::from(format!("{error:#}"));
2950                    *slot.state.lock().await = RunnerSessionSlotState::Failed(message);
2951                    slot.initialized.notify_waiters();
2952                    let mut sessions = runner_sessions.lock().await;
2953                    let is_current = sessions
2954                        .get(&thread_id)
2955                        .is_some_and(|current| Arc::ptr_eq(current, &slot));
2956                    if is_current {
2957                        sessions.remove(&thread_id);
2958                    }
2959                }
2960            }
2961        }));
2962    }
2963
2964    async fn evict_runner_session(&self, thread_id: &ThreadId) {
2965        self.runner_sessions.lock().await.remove(thread_id);
2966    }
2967
2968    async fn cache_runner_session(
2969        &self,
2970        thread_id: &ThreadId,
2971        destination: RunnerDestination,
2972        session: Arc<dyn RemoteRunnerSession>,
2973    ) {
2974        let cached = CachedRunnerSession {
2975            destination,
2976            session,
2977        };
2978        self.runner_sessions.lock().await.insert(
2979            thread_id.clone(),
2980            Arc::new(RunnerSessionSlot::ready(cached)),
2981        );
2982    }
2983
2984    /// Read a thread's explicit runner binding without creating or resuming a
2985    /// session. Tool routing uses this before local previews so runner-backed
2986    /// and fail-closed hosted calls never inspect the host workspace.
2987    pub(crate) async fn runner_binding_for_thread(
2988        &self,
2989        thread_id: &ThreadId,
2990    ) -> anyhow::Result<Option<ThreadRunnerBinding>> {
2991        let Some(store) = &self.thread_store else {
2992            return Ok(None);
2993        };
2994        Ok(store
2995            .load_thread_metadata(thread_id)
2996            .await?
2997            .and_then(|metadata| metadata.runner_binding))
2998    }
2999
3000    /// Resolve a previously-read binding into a live remote workspace. The
3001    /// session remains lazy: callers invoke this only after policy approval.
3002    pub(crate) async fn remote_workspace_for_binding(
3003        &self,
3004        thread_id: &ThreadId,
3005        binding: ThreadRunnerBinding,
3006    ) -> anyhow::Result<Arc<RemoteWorkspace>> {
3007        let session = self
3008            .runner_session_for_thread(thread_id)
3009            .await?
3010            .map(|(_, session)| session)
3011            .ok_or_else(|| {
3012                anyhow::anyhow!("runner-bound thread {thread_id} has no runner session")
3013            })?;
3014        Ok(Arc::new(RemoteWorkspace {
3015            session,
3016            root: binding.workspace,
3017            read_roots: binding.read_roots,
3018        }))
3019    }
3020
3021    pub(crate) async fn runner_tool_execution_lock(
3022        &self,
3023        session: &dyn RemoteRunnerSession,
3024    ) -> Arc<RunnerToolExecutionMutex> {
3025        let state = session.state();
3026        let key = (state.provider_id, state.session_id);
3027        let mut locks = self.runner_tool_execution_locks.lock().await;
3028        if let Some(lock) = locks.get(&key).and_then(Weak::upgrade) {
3029            return lock;
3030        }
3031
3032        locks.retain(|_, lock| lock.strong_count() > 0);
3033        let lock = Arc::new(Mutex::new(()));
3034        locks.insert(key, Arc::downgrade(&lock));
3035        lock
3036    }
3037
3038    async fn persist_runner_state(
3039        &self,
3040        thread_id: &ThreadId,
3041        runner: Option<&(RunnerDestination, Arc<dyn RemoteRunnerSession>)>,
3042    ) -> anyhow::Result<()> {
3043        Self::persist_runner_state_in_store(self.thread_store.as_ref(), thread_id, runner).await
3044    }
3045
3046    async fn persist_runner_state_in_store(
3047        store: Option<&Arc<dyn ThreadStore>>,
3048        thread_id: &ThreadId,
3049        runner: Option<&(RunnerDestination, Arc<dyn RemoteRunnerSession>)>,
3050    ) -> anyhow::Result<()> {
3051        let Some((destination, session)) = runner else {
3052            return Ok(());
3053        };
3054        let Some(store) = store else {
3055            return Ok(());
3056        };
3057        let Some(snapshot) = store.load_thread(thread_id).await? else {
3058            return Ok(());
3059        };
3060        let Some(mut metadata) = snapshot.metadata else {
3061            return Ok(());
3062        };
3063        metadata.runner_destination = Some(destination.clone());
3064        metadata.runner_state = Some(session.state());
3065        metadata.updated_at = OffsetDateTime::now_utc();
3066        store.update_thread_metadata(metadata).await?;
3067        Ok(())
3068    }
3069
3070    fn remote_runner_provider_by_id(
3071        &self,
3072        provider_id: &str,
3073    ) -> Option<Arc<dyn RemoteRunnerProvider>> {
3074        self.registry
3075            .remote_runner_providers
3076            .iter()
3077            .find(|provider| provider.id() == provider_id)
3078            .cloned()
3079    }
3080
3081    /// Resolve the live runner session for a thread along with its provider,
3082    /// failing clearly when the thread is not runner-bound.
3083    async fn thread_runner_session(
3084        &self,
3085        thread_id: &ThreadId,
3086    ) -> anyhow::Result<(
3087        RunnerDestination,
3088        Arc<dyn RemoteRunnerProvider>,
3089        Arc<dyn RemoteRunnerSession>,
3090    )> {
3091        let Some((destination, session)) = self.runner_session_for_thread(thread_id).await? else {
3092            anyhow::bail!("thread {thread_id} is not bound to a remote runner");
3093        };
3094        let provider = self
3095            .remote_runner_provider_by_id(&destination.provider_id)
3096            .ok_or_else(|| {
3097                anyhow::anyhow!(
3098                    "remote runner provider {:?} is not installed",
3099                    destination.provider_id
3100                )
3101            })?;
3102        Ok((destination, provider, session))
3103    }
3104
3105    /// Pause a runner-bound thread's session toward standby and persist the
3106    /// post-pause state. Errors if the provider is not pausable.
3107    pub async fn pause_thread_runner(
3108        &self,
3109        thread_id: &ThreadId,
3110    ) -> anyhow::Result<RunnerSessionState> {
3111        let (destination, provider, session) = self.thread_runner_session(thread_id).await?;
3112        anyhow::ensure!(
3113            provider.capabilities().pausable,
3114            "remote runner provider {:?} does not support pausing",
3115            destination.provider_id
3116        );
3117        let state = session.pause().await?;
3118        self.persist_runner_state(thread_id, Some(&(destination, session)))
3119            .await?;
3120        Ok(state)
3121    }
3122
3123    /// Resume (wake) a runner-bound thread's paused session and persist state.
3124    pub async fn resume_thread_runner(
3125        &self,
3126        thread_id: &ThreadId,
3127    ) -> anyhow::Result<RunnerSessionState> {
3128        let (destination, _provider, session) = self.thread_runner_session(thread_id).await?;
3129        let state = session.resume().await?;
3130        self.persist_runner_state(thread_id, Some(&(destination, session)))
3131            .await?;
3132        Ok(state)
3133    }
3134
3135    /// Detach a runner-bound thread's session: persist the durable, rejoinable
3136    /// state and leave the remote sandbox alive. Errors if not detachable.
3137    pub async fn detach_thread_runner(
3138        &self,
3139        thread_id: &ThreadId,
3140    ) -> anyhow::Result<RunnerSessionState> {
3141        let (destination, provider, session) = self.thread_runner_session(thread_id).await?;
3142        anyhow::ensure!(
3143            provider.capabilities().detachable,
3144            "remote runner provider {:?} does not support detaching",
3145            destination.provider_id
3146        );
3147        let state = session.detach().await?;
3148        // Persist the detached state explicitly so a later turn or process
3149        // rejoins the same sandbox instead of provisioning a new one.
3150        if let Some(store) = &self.thread_store
3151            && let Some(snapshot) = store.load_thread(thread_id).await?
3152            && let Some(mut metadata) = snapshot.metadata
3153        {
3154            metadata.runner_destination = Some(destination);
3155            metadata.runner_state = Some(state.clone());
3156            metadata.updated_at = OffsetDateTime::now_utc();
3157            store.update_thread_metadata(metadata).await?;
3158        }
3159        self.evict_runner_session(thread_id).await;
3160        Ok(state)
3161    }
3162
3163    /// Rejoin a previously created sandbox from a thread's persisted runner
3164    /// state without provisioning a new one. An optional `sandbox` overrides the
3165    /// persisted sandbox name (recovery by name). Persists refreshed state.
3166    pub async fn rejoin_thread_runner(
3167        &self,
3168        thread_id: &ThreadId,
3169        sandbox: Option<String>,
3170    ) -> anyhow::Result<RunnerSessionState> {
3171        let store = self
3172            .thread_store
3173            .as_ref()
3174            .ok_or_else(|| anyhow::anyhow!("thread store is required to rejoin a runner"))?;
3175        let metadata = store
3176            .load_thread_metadata(thread_id)
3177            .await?
3178            .ok_or_else(|| anyhow::anyhow!("thread {thread_id} has no metadata"))?;
3179        let destination = metadata
3180            .runner_binding
3181            .as_ref()
3182            .map(|binding| binding.destination.clone())
3183            .or_else(|| metadata.runner_destination.clone())
3184            .ok_or_else(|| anyhow::anyhow!("thread {thread_id} is not bound to a remote runner"))?;
3185        let mut state = metadata
3186            .runner_state
3187            .clone()
3188            .ok_or_else(|| anyhow::anyhow!("thread {thread_id} has no persisted runner state"))?;
3189        if let Some(sandbox) = sandbox
3190            && let Some(object) = state.metadata.as_object_mut()
3191        {
3192            object.insert("sandbox_name".to_string(), serde_json::Value::from(sandbox));
3193        }
3194        let provider = self
3195            .remote_runner_provider_by_id(&destination.provider_id)
3196            .ok_or_else(|| {
3197                anyhow::anyhow!(
3198                    "remote runner provider {:?} is not installed",
3199                    destination.provider_id
3200                )
3201            })?;
3202        let session = provider.rejoin_session(state).await?;
3203        self.persist_runner_state(thread_id, Some(&(destination.clone(), session.clone())))
3204            .await?;
3205        self.cache_runner_session(thread_id, destination, session.clone())
3206            .await;
3207        Ok(session.state())
3208    }
3209
3210    pub(crate) async fn record_thread_usage_metadata(
3211        &self,
3212        thread_id: &ThreadId,
3213        usage: &TokenUsage,
3214    ) -> anyhow::Result<()> {
3215        if usage.is_empty() {
3216            return Ok(());
3217        }
3218        let Some(store) = &self.thread_store else {
3219            return Ok(());
3220        };
3221        let Some(snapshot) = store.load_thread(thread_id).await? else {
3222            return Ok(());
3223        };
3224        let Some(mut metadata) = snapshot.metadata else {
3225            return Ok(());
3226        };
3227        metadata
3228            .usage
3229            .get_or_insert_with(ThreadUsageMetadata::default)
3230            .add_token_usage(usage);
3231        metadata.updated_at = OffsetDateTime::now_utc();
3232        store.update_thread_metadata(metadata).await?;
3233        Ok(())
3234    }
3235
3236    pub fn start_turn(
3237        self: &Arc<Self>,
3238        mut req: StartTurnRequest,
3239    ) -> BoxFuture<'_, anyhow::Result<TurnId>> {
3240        Box::pin(async move {
3241            let _thread_admission = self.thread_admission(&req.thread_id).await;
3242            let turn_admission = self.turn_admission.lock().await;
3243            self.ensure_execution_authority()?;
3244            anyhow::ensure!(
3245                self.accepting_turns.load(Ordering::Acquire),
3246                "runtime is quiescing and cannot accept new turns"
3247            );
3248            req.workspace = validate_thread_workspace(&req.workspace)?;
3249            let team_member = self.teams.member_for_thread(&req.thread_id).await;
3250            let cfg = self.config.read().await.clone();
3251            let provider = req
3252                .provider_override
3253                .clone()
3254                .unwrap_or_else(|| cfg.default_provider.clone());
3255            self.engine_for(&provider)?;
3256            let turn_id = uuid::Uuid::new_v4().to_string();
3257            let mut initial_mailbox_ack = None;
3258            if let Some((team_id, member)) = &team_member {
3259                let pending = self
3260                    .teams
3261                    .reserve_pending_mailbox_messages(team_id, &member.id, &turn_id)
3262                    .await?;
3263                if !pending.is_empty()
3264                    && let Some(team) = self.read_team(team_id).await
3265                {
3266                    let mailbox = format_mailbox_messages(&team, &pending);
3267                    req.message = if req.message.trim().is_empty() {
3268                        mailbox
3269                    } else {
3270                        format!("{mailbox}\n\n[Direct task input]\n{}", req.message)
3271                    };
3272                    initial_mailbox_ack = Some(MailboxDeliveryAck {
3273                        team_id: team_id.clone(),
3274                        message_ids: pending.iter().map(|message| message.id.clone()).collect(),
3275                    });
3276                }
3277            }
3278            let (abort_handle, abort_registration) = AbortHandle::new_pair();
3279            let drain = Arc::new(TurnDrainHandle {
3280                thread_id: req.thread_id.clone(),
3281                interrupt_requested: AtomicBool::new(false),
3282                interrupt_reason: Mutex::new(None),
3283                completed: AtomicBool::new(false),
3284                completed_notify: Notify::new(),
3285            });
3286            let (steer_changed, steering) = tokio::sync::watch::channel(0);
3287            let active = ActiveTurnHandle {
3288                thread_id: req.thread_id.clone(),
3289                abort: abort_handle,
3290                steers: Arc::new(Mutex::new(Vec::new())),
3291                steer_changed,
3292                drain,
3293            };
3294            self.active_turns
3295                .write()
3296                .await
3297                .insert(turn_id.clone(), active);
3298            self.record_turn_lifecycle(
3299                req.thread_id.clone(),
3300                turn_id.clone(),
3301                TurnLifecycleState::Running,
3302                TurnCleanupState::NotRequested,
3303                None,
3304            )
3305            .await;
3306            self.active_turn_contexts.write().await.insert(
3307                turn_id.clone(),
3308                InheritedTurnContext {
3309                    workspace: req.workspace.clone(),
3310                    instructions: req.instructions.clone(),
3311                    developer_context: req.developer_context.clone(),
3312                },
3313            );
3314            if let Some((team_id, member)) = team_member {
3315                let updated = match self
3316                    .teams
3317                    .update_member(&team_id, &member.id, |member| {
3318                        member.current_turn_id = Some(turn_id.clone());
3319                        member.status = TeamMemberStatus::Running;
3320                        member.final_message = None;
3321                        member.terminal_error = None;
3322                    })
3323                    .await
3324                {
3325                    Ok(updated) => updated,
3326                    Err(error) => {
3327                        self.active_turns.write().await.remove(&turn_id);
3328                        self.active_turn_contexts.write().await.remove(&turn_id);
3329                        self.teams
3330                            .release_mailbox_reservations_for_turn(&turn_id)
3331                            .await;
3332                        return Err(error);
3333                    }
3334                };
3335                if let Some(member) = updated
3336                    .members
3337                    .into_iter()
3338                    .find(|candidate| candidate.id == member.id)
3339                {
3340                    self.emit(RoderEvent::TeamMemberStatusChanged(
3341                        TeamMemberStatusChanged {
3342                            team_id,
3343                            member_id: member.id,
3344                            member_thread_id: member.thread_id,
3345                            status: TeamMemberStatus::Running,
3346                            timestamp: OffsetDateTime::now_utc(),
3347                        },
3348                    ))
3349                    .await;
3350                }
3351            }
3352            let runtime = Arc::clone(self);
3353            let turn_req = req;
3354            let thread_id_for_task = turn_req.thread_id.clone();
3355            let turn_id_for_task = turn_id.clone();
3356            tokio::spawn(async move {
3357                let result = Abortable::new(
3358                    runtime.run_turn(
3359                        turn_req,
3360                        turn_id_for_task.clone(),
3361                        initial_mailbox_ack,
3362                        steering,
3363                    ),
3364                    abort_registration,
3365                )
3366                .await;
3367                /*
3368                 * A failed sibling in a parallel tool batch drops in-flight external tool
3369                 * futures (`try_join_all` in `route_tool_calls`), stranding their
3370                 * `pending_external_tool_calls` entries. Sweep before reporting the turn
3371                 * outcome so every `thread/toolExecutionRequested` gets a terminal
3372                 * resolution; on clean completion the map holds nothing for this turn.
3373                 */
3374                runtime
3375                    .cancel_pending_external_tool_calls_for_turn(&turn_id_for_task)
3376                    .await;
3377                let completed = matches!(&result, Ok(Ok(TurnRunOutcome::Completed)));
3378                match &result {
3379                    Ok(Err(err)) => {
3380                        let (cleanup, ownership) =
3381                            runtime.await_provider_turn_cleanup(&turn_id_for_task).await;
3382                        // This wrapper owns terminal failure emission for returned errors.
3383                        runtime
3384                            .emit(RoderEvent::TurnFailed(TurnFailed {
3385                                thread_id: thread_id_for_task.clone(),
3386                                turn_id: turn_id_for_task.clone(),
3387                                error: err.to_string(),
3388                                error_kind: err
3389                                    .downcast_ref::<roder_api::provider_error::ProviderFailure>()
3390                                    .map(|failure| failure.kind.retry_cause().to_string()),
3391                                usage: None,
3392                                timestamp: OffsetDateTime::now_utc(),
3393                            }))
3394                            .await;
3395                        runtime
3396                            .record_turn_lifecycle_with_ownership(
3397                                thread_id_for_task.clone(),
3398                                turn_id_for_task.clone(),
3399                                TurnLifecycleState::Failed,
3400                                cleanup,
3401                                Some(TurnLifecycleReason::ProviderFailure),
3402                                ownership,
3403                            )
3404                            .await;
3405                        let _ = runtime
3406                            .complete_team_member_turn_with_result(
3407                                &thread_id_for_task,
3408                                &turn_id_for_task,
3409                                TeamMemberStatus::Failed,
3410                                None,
3411                                Some(err.to_string()),
3412                            )
3413                            .await;
3414                    }
3415                    Ok(Ok(TurnRunOutcome::Stopped)) => {
3416                        let (cleanup, ownership) =
3417                            runtime.await_provider_turn_cleanup(&turn_id_for_task).await;
3418                        runtime
3419                            .record_turn_lifecycle_with_ownership(
3420                                thread_id_for_task.clone(),
3421                                turn_id_for_task.clone(),
3422                                TurnLifecycleState::Failed,
3423                                cleanup,
3424                                Some(TurnLifecycleReason::ProviderFailure),
3425                                ownership,
3426                            )
3427                            .await;
3428                        let _ = runtime
3429                            .complete_team_member_turn_with_result(
3430                                &thread_id_for_task,
3431                                &turn_id_for_task,
3432                                TeamMemberStatus::Failed,
3433                                None,
3434                                Some("turn stopped before completion".to_string()),
3435                            )
3436                            .await;
3437                    }
3438                    Err(_) => {
3439                        let reason = if let Some(handle) = runtime
3440                            .turn_drains
3441                            .read()
3442                            .await
3443                            .get(&turn_id_for_task)
3444                            .cloned()
3445                        {
3446                            handle
3447                                .interrupt_reason
3448                                .lock()
3449                                .await
3450                                .unwrap_or(TurnLifecycleReason::RuntimeFailure)
3451                        } else {
3452                            TurnLifecycleReason::RuntimeFailure
3453                        };
3454                        let (cleanup, ownership) =
3455                            runtime.await_provider_turn_cleanup(&turn_id_for_task).await;
3456                        runtime
3457                            .record_turn_lifecycle_with_ownership(
3458                                thread_id_for_task.clone(),
3459                                turn_id_for_task.clone(),
3460                                TurnLifecycleState::Interrupted,
3461                                cleanup,
3462                                Some(reason),
3463                                ownership,
3464                            )
3465                            .await;
3466                        runtime
3467                            .emit(RoderEvent::TurnInterrupted(TurnInterrupted {
3468                                thread_id: thread_id_for_task.clone(),
3469                                turn_id: turn_id_for_task.clone(),
3470                                timestamp: OffsetDateTime::now_utc(),
3471                            }))
3472                            .await;
3473                        let _ = runtime
3474                            .complete_team_member_turn_with_result(
3475                                &thread_id_for_task,
3476                                &turn_id_for_task,
3477                                TeamMemberStatus::Interrupted,
3478                                None,
3479                                None,
3480                            )
3481                            .await;
3482                    }
3483                    Ok(Ok(TurnRunOutcome::Completed)) => {}
3484                }
3485                runtime
3486                    .teams
3487                    .release_mailbox_reservations_for_turn(&turn_id_for_task)
3488                    .await;
3489                runtime.active_turns.write().await.remove(&turn_id_for_task);
3490                if let Some(drain) = runtime.turn_drains.write().await.remove(&turn_id_for_task) {
3491                    drain.completed.store(true, Ordering::Release);
3492                    drain.completed_notify.notify_waiters();
3493                }
3494                runtime
3495                    .active_turn_selections
3496                    .write()
3497                    .await
3498                    .remove(&turn_id_for_task);
3499                runtime
3500                    .active_turn_contexts
3501                    .write()
3502                    .await
3503                    .remove(&turn_id_for_task);
3504                if !completed {
3505                    // Completion paths above consume registered provider cleanup
3506                    // handles. A setup failure before an engine can stream has no
3507                    // such handle; this is a harmless final defensive sweep.
3508                    let _ = runtime.provider_turn_cleanups.lock().map(|mut cleanups| {
3509                        cleanups.remove(&turn_id_for_task);
3510                    });
3511                }
3512                runtime.active_turns_changed.notify_waiters();
3513                if completed {
3514                    let _ = runtime
3515                        .continue_active_goal_after_turn(thread_id_for_task)
3516                        .await;
3517                }
3518            });
3519            drop(turn_admission);
3520            Ok(turn_id)
3521        })
3522    }
3523
3524    pub(crate) async fn has_active_turn_for_thread(&self, thread_id: &ThreadId) -> bool {
3525        self.active_turns
3526            .read()
3527            .await
3528            .values()
3529            .any(|handle| &handle.thread_id == thread_id)
3530    }
3531
3532    async fn ensure_codex_v2_team_capacity(
3533        &self,
3534        team: &TeamState,
3535        resuming_member_id: &str,
3536        candidate_model: Option<&str>,
3537    ) -> anyhow::Result<()> {
3538        if !is_codex_v2_team(team)
3539            && !candidate_model
3540                .is_some_and(|model| model_supports_reasoning_effort(model, REASONING_ULTRA))
3541        {
3542            return Ok(());
3543        }
3544        let active_thread_ids = self
3545            .active_turns
3546            .read()
3547            .await
3548            .values()
3549            .map(|handle| handle.thread_id.clone())
3550            .collect::<std::collections::HashSet<_>>();
3551        let resident_threads = 1 + team
3552            .members
3553            .iter()
3554            .filter(|member| {
3555                member.role != roder_api::teams::TeamMemberRole::Lead
3556                    && member.id != resuming_member_id
3557                    && active_thread_ids.contains(&member.thread_id)
3558            })
3559            .count();
3560        anyhow::ensure!(
3561            resident_threads < codex_v2::CODEX_V2_MAX_RESIDENT_TEAM_THREADS,
3562            "agent thread limit reached for this Codex V2 team: maximum {} resident threads (the lead plus 3 running subagents)",
3563            codex_v2::CODEX_V2_MAX_RESIDENT_TEAM_THREADS
3564        );
3565        Ok(())
3566    }
3567
3568    /// Number of currently running turns across all threads. Hosted runtime
3569    /// pools use this to avoid evicting tenants with active work.
3570    pub async fn active_turn_count(&self) -> usize {
3571        self.active_turns.read().await.len()
3572    }
3573
3574    pub async fn active_turn_for_thread(&self, thread_id: &ThreadId) -> Option<TurnId> {
3575        self.active_turns
3576            .read()
3577            .await
3578            .iter()
3579            .find_map(|(turn_id, handle)| (&handle.thread_id == thread_id).then(|| turn_id.clone()))
3580    }
3581
3582    pub async fn thread_activity(&self, thread_id: &ThreadId) -> ThreadActivity {
3583        let active_turn_id = self.active_turn_for_thread(thread_id).await;
3584        let draining_turn_id = if active_turn_id.is_none() {
3585            self.turn_drains
3586                .read()
3587                .await
3588                .iter()
3589                .find_map(|(turn_id, drain)| {
3590                    (&drain.thread_id == thread_id).then(|| turn_id.clone())
3591                })
3592        } else {
3593            None
3594        };
3595        let Some(active_turn_id) = active_turn_id.or(draining_turn_id) else {
3596            return ThreadActivity::default();
3597        };
3598
3599        let mut active_flags = Vec::new();
3600        {
3601            let pending_approvals = self.pending_tool_approvals.lock().await;
3602            if pending_approvals
3603                .values()
3604                .any(|pending| &pending.thread_id == thread_id && pending.turn_id == active_turn_id)
3605            {
3606                active_flags.push("approvalRequired".to_string());
3607            }
3608        }
3609        {
3610            let pending_inputs = self.pending_user_inputs.lock().await;
3611            if pending_inputs
3612                .values()
3613                .any(|pending| &pending.thread_id == thread_id && pending.turn_id == active_turn_id)
3614            {
3615                active_flags.push("userInputRequired".to_string());
3616            }
3617        }
3618        {
3619            let pending_external = self.pending_external_tool_calls.lock().await;
3620            if pending_external
3621                .values()
3622                .any(|pending| &pending.thread_id == thread_id && pending.turn_id == active_turn_id)
3623            {
3624                active_flags.push("externalToolPending".to_string());
3625            }
3626        }
3627        let interrupting = self
3628            .active_turns
3629            .read()
3630            .await
3631            .get(&active_turn_id)
3632            .is_some_and(|handle| handle.drain.interrupt_requested.load(Ordering::Acquire))
3633            || self
3634                .turn_drains
3635                .read()
3636                .await
3637                .get(&active_turn_id)
3638                .is_some_and(|drain| drain.interrupt_requested.load(Ordering::Acquire));
3639        if interrupting {
3640            active_flags.push("interrupting".to_string());
3641        }
3642        if self.pending_plan_exit().await.is_some_and(|pending| {
3643            &pending.thread_id == thread_id && pending.turn_id == active_turn_id
3644        }) {
3645            active_flags.push("planExitRequired".to_string());
3646        }
3647
3648        ThreadActivity {
3649            active_turn_id: Some(active_turn_id),
3650            active_flags,
3651        }
3652    }
3653
3654    pub async fn interrupt_turn(&self, thread_id: ThreadId, turn_id: TurnId) -> anyhow::Result<()> {
3655        self.interrupt_turn_with_reason(thread_id, turn_id, TurnLifecycleReason::UserInterrupt)
3656            .await
3657    }
3658
3659    async fn interrupt_turn_with_reason(
3660        &self,
3661        thread_id: ThreadId,
3662        turn_id: TurnId,
3663        reason: TurnLifecycleReason,
3664    ) -> anyhow::Result<()> {
3665        let handle = self.active_turns.write().await.remove(&turn_id);
3666        if let Some(handle) = handle {
3667            if handle
3668                .drain
3669                .interrupt_requested
3670                .swap(true, Ordering::AcqRel)
3671            {
3672                return Ok(());
3673            }
3674            *handle.drain.interrupt_reason.lock().await = Some(reason);
3675            self.turn_drains
3676                .write()
3677                .await
3678                .insert(turn_id.clone(), handle.drain.clone());
3679            let ownership = self.provider_cleanup_ownership(&turn_id);
3680            self.record_turn_lifecycle_with_ownership(
3681                thread_id.clone(),
3682                turn_id.clone(),
3683                TurnLifecycleState::InterruptRequested,
3684                TurnCleanupState::Requested,
3685                Some(reason),
3686                ownership,
3687            )
3688            .await;
3689            handle.abort.abort();
3690            self.active_turns_changed.notify_waiters();
3691        }
3692        self.active_turn_selections.write().await.remove(&turn_id);
3693        self.cancel_pending_external_tool_calls_for_turn(&turn_id)
3694            .await;
3695        Ok(())
3696    }
3697
3698    pub async fn steer_turn(
3699        &self,
3700        thread_id: ThreadId,
3701        turn_id: TurnId,
3702        message: String,
3703        images: Vec<InputImage>,
3704    ) -> anyhow::Result<()> {
3705        self.enqueue_turn_steer(thread_id, turn_id, message, images, None)
3706            .await
3707    }
3708
3709    pub(crate) async fn has_pending_turn_steers(&self, turn_id: &TurnId) -> bool {
3710        let active = self.active_turns.read().await.get(turn_id).cloned();
3711        let Some(active) = active else {
3712            return false;
3713        };
3714        let has_pending = !active.steers.lock().await.is_empty();
3715        has_pending
3716    }
3717
3718    async fn steer_turn_with_mailbox_ack(
3719        &self,
3720        thread_id: ThreadId,
3721        turn_id: TurnId,
3722        message: String,
3723        message_ids: Vec<String>,
3724        team_id: TeamId,
3725    ) -> anyhow::Result<()> {
3726        self.enqueue_turn_steer(
3727            thread_id,
3728            turn_id,
3729            message,
3730            Vec::new(),
3731            Some(MailboxDeliveryAck {
3732                team_id,
3733                message_ids,
3734            }),
3735        )
3736        .await
3737    }
3738
3739    async fn deliver_pending_team_mailbox(
3740        &self,
3741        team_id: &str,
3742        member_id: &str,
3743        member_thread_id: ThreadId,
3744        turn_id: TurnId,
3745    ) -> anyhow::Result<()> {
3746        let pending = self
3747            .teams
3748            .reserve_pending_mailbox_messages(team_id, member_id, &turn_id)
3749            .await?;
3750        if pending.is_empty() {
3751            return Ok(());
3752        }
3753        let message_ids = pending
3754            .iter()
3755            .map(|message| message.id.clone())
3756            .collect::<Vec<_>>();
3757        let team = self
3758            .read_team(team_id)
3759            .await
3760            .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
3761        if let Err(error) = self
3762            .steer_turn_with_mailbox_ack(
3763                member_thread_id,
3764                turn_id.clone(),
3765                format_mailbox_messages(&team, &pending),
3766                message_ids.clone(),
3767                team_id.to_string(),
3768            )
3769            .await
3770        {
3771            self.teams
3772                .release_mailbox_reservations(&turn_id, &message_ids)
3773                .await;
3774            return Err(error);
3775        }
3776        Ok(())
3777    }
3778
3779    async fn enqueue_turn_steer(
3780        &self,
3781        thread_id: ThreadId,
3782        turn_id: TurnId,
3783        message: String,
3784        images: Vec<InputImage>,
3785        mailbox_ack: Option<MailboxDeliveryAck>,
3786    ) -> anyhow::Result<()> {
3787        let message = message.trim().to_string();
3788        if message.is_empty() && images.is_empty() {
3789            return Ok(());
3790        }
3791
3792        let Some(active) = self.active_turns.read().await.get(&turn_id).cloned() else {
3793            anyhow::bail!("no active turn to steer");
3794        };
3795        anyhow::ensure!(
3796            active.thread_id == thread_id,
3797            "turn does not belong to thread"
3798        );
3799        {
3800            let mut steers = active.steers.lock().await;
3801            steers.push(QueuedTurnSteer {
3802                message: UserMessage::with_images(message.clone(), images),
3803                mailbox_ack,
3804            });
3805            active
3806                .steer_changed
3807                .send_modify(|generation| *generation = generation.wrapping_add(1));
3808        }
3809        self.emit(RoderEvent::TurnSteered(TurnSteered {
3810            thread_id,
3811            turn_id,
3812            message,
3813            timestamp: OffsetDateTime::now_utc(),
3814        }))
3815        .await;
3816        Ok(())
3817    }
3818
3819    pub async fn tool_specs(&self) -> Vec<roder_api::tools::ToolSpec> {
3820        let cfg = self.config.read().await;
3821        let model_profile =
3822            model_profile_for_provider_model(&cfg, &cfg.default_provider, &cfg.default_model);
3823        self.filtered_tool_specs(
3824            &cfg,
3825            &cfg.default_model,
3826            &cfg.default_provider,
3827            model_profile.as_ref(),
3828            &[],
3829            &[],
3830        )
3831    }
3832
3833    pub fn subagent_definitions(&self) -> Vec<SubagentDefinition> {
3834        self.registry
3835            .subagent_dispatchers
3836            .iter()
3837            .flat_map(|dispatcher| dispatcher.definitions())
3838            .collect()
3839    }
3840
3841    async fn run_turn(
3842        self: &Arc<Self>,
3843        req: StartTurnRequest,
3844        turn_id: TurnId,
3845        initial_mailbox_ack: Option<MailboxDeliveryAck>,
3846        mut steering: tokio::sync::watch::Receiver<u64>,
3847    ) -> anyhow::Result<TurnRunOutcome> {
3848        self.ensure_execution_authority()?;
3849        let turn_started_at = OffsetDateTime::now_utc();
3850        self.emit(RoderEvent::TurnStarted(TurnStarted {
3851            thread_id: req.thread_id.clone(),
3852            turn_id: turn_id.clone(),
3853            runtime_profile: self.config.read().await.runtime_profile,
3854            timestamp: turn_started_at,
3855        }))
3856        .await;
3857        self.persist_turn_item(
3858            &req.thread_id,
3859            &turn_id,
3860            &TranscriptItem::UserMessage(UserMessage::with_images(
3861                req.message.clone(),
3862                req.images.clone(),
3863            )),
3864        )
3865        .await?;
3866        crate::hooks::run_lifecycle(
3867            self,
3868            &req.thread_id,
3869            &turn_id,
3870            Some(&req.workspace),
3871            "UserPromptSubmit",
3872            None,
3873            serde_json::json!({"prompt": req.message}),
3874        )
3875        .await;
3876        if let Some(ack) = initial_mailbox_ack {
3877            self.teams
3878                .mark_mailbox_messages_delivered(&ack.team_id, &turn_id, &ack.message_ids)
3879                .await?;
3880        }
3881
3882        let mut cfg = self.config.read().await.clone();
3883        let runtime_profile = cfg.runtime_profile;
3884        let turn_deadline = turn_deadline_for_config(&cfg);
3885        let deadline_finalization_reserve =
3886            crate::deadline_policy::finalization_reserve_seconds(cfg.turn_deadline_seconds);
3887        let selection_mode = self.selection_mode_for_thread(&req.thread_id).await?;
3888        let concrete_selection = selection_mode
3889            .as_ref()
3890            .map(ModelSelectionMode::concrete_selection);
3891        let default_provider = req
3892            .provider_override
3893            .clone()
3894            .or_else(|| {
3895                concrete_selection
3896                    .as_ref()
3897                    .map(|selection| selection.provider.clone())
3898            })
3899            .unwrap_or(cfg.default_provider.clone());
3900        let default_model = req
3901            .model_override
3902            .clone()
3903            .or_else(|| {
3904                concrete_selection
3905                    .as_ref()
3906                    .map(|selection| selection.model.clone())
3907            })
3908            .unwrap_or(cfg.default_model.clone());
3909        if let Some(reasoning) = req.reasoning_override.as_deref().or_else(|| {
3910            selection_mode
3911                .as_ref()
3912                .and_then(ModelSelectionMode::reasoning)
3913        }) {
3914            validate_reasoning_effort(&default_model, reasoning)?;
3915            cfg.reasoning = Some(reasoning.to_string());
3916        }
3917        let turn_has_concrete_model_override =
3918            req.provider_override.is_some() || req.model_override.is_some();
3919        let (turn_inference_router, turn_inference_router_profile) = match &selection_mode {
3920            Some(ModelSelectionMode::Auto {
3921                router_id, profile, ..
3922            }) if !turn_has_concrete_model_override => (
3923                RuntimeInferenceRouterConfig {
3924                    enabled: true,
3925                    router_id: Some(router_id.clone()),
3926                },
3927                profile.clone(),
3928            ),
3929            _ => (RuntimeInferenceRouterConfig::disabled(), None),
3930        };
3931        let mut provider = default_provider.clone();
3932        let mut model = default_model.clone();
3933        let mut model_profile = model_profile_for_provider_model(&cfg, &provider, &model);
3934        let workspace = req.workspace.clone();
3935        let compaction_generation_at_start = self.compaction_generation(&req.thread_id);
3936        let mut transcript = self.transcript_for_turn(&req, &turn_id, &model).await?;
3937        let mut compacted_this_turn =
3938            self.compaction_generation(&req.thread_id) != compaction_generation_at_start;
3939        let effective_policy_mode = self.effective_policy_mode_for_thread(&req.thread_id).await;
3940        let agent_swarm_mode_active = self
3941            .effective_agent_swarm_mode_for_thread(&req.thread_id)
3942            .await;
3943        let ultra_mode_active = self.effective_ultra_mode_for_thread(&req.thread_id).await;
3944        let thread_overrides = self.thread_turn_overrides(&req.thread_id).await?;
3945        let mut final_assistant_text = String::new();
3946        let mut final_provider_metadata = None;
3947        let mut exhausted_tool_rounds = true;
3948        let mut verification_gate =
3949            VerificationGateState::new(req.message.clone(), runtime_profile);
3950        let mut speed_policy = SpeedPolicyState::default();
3951        let mut reliability = TurnReliabilityState::default();
3952        let mut turn_usage = TokenUsage::default();
3953        // Overwritten on every inference step's Completed event so only the
3954        // terminal step's stop reason survives (mid-turn tool_use steps must
3955        // not leak onto turn/completed).
3956        let mut turn_finish_reason: Option<String> = None;
3957        let mut deadline_finalization_requested = false;
3958        let mut deadline_scoreable_completion_requested = false;
3959        let mut task_ledger_completion_reminders = 0_u8;
3960        let mut task_ledger_scoreable_checkpoints = 0_u8;
3961        let mut empty_tool_call_nudges_used = 0_u32;
3962        let mut provider_stream_retry_attempts = 0_u32;
3963        let mut context_limit_recovery_attempts = 0_u32;
3964        let mut routing_candidates = None;
3965        let routing_transcript_start = transcript.len().saturating_sub(1);
3966        let mut routing_escalations = 0_u32;
3967        let mut model_switch_summary_selection = None::<ModelSelection>;
3968
3969        'tool_rounds: for round_index in 0..MAX_TOOL_ROUNDS_PER_TURN {
3970            if let Some(deadline) = turn_deadline
3971                && deadline_expired(deadline)
3972            {
3973                self.fail_turn_due_to_deadline(&req.thread_id, &turn_id, deadline, &transcript)
3974                    .await?;
3975                return Ok(TurnRunOutcome::Stopped);
3976            }
3977            let steers = self.drain_turn_steers(&turn_id, &mut steering).await;
3978            self.append_steers(&req, &turn_id, &mut transcript, steers)
3979                .await?;
3980            if runtime_profile == RuntimeProfile::Eval
3981                && let Some(remaining) = crate::deadline_policy::should_start_finalization(
3982                    turn_deadline,
3983                    deadline_finalization_reserve,
3984                    deadline_finalization_requested || deadline_scoreable_completion_requested,
3985                )
3986            {
3987                if req.task_ledger_required
3988                    && task_ledger_completion_reminders < TASK_LEDGER_COMPLETION_REMINDER_LIMIT
3989                    && let Some(prompt) = task_ledger_completion_prompt(&transcript)
3990                {
3991                    task_ledger_completion_reminders += 1;
3992                    deadline_scoreable_completion_requested = true;
3993                    let item = TranscriptItem::UserMessage(UserMessage::text(
3994                        task_ledger_deadline_completion_prompt(
3995                            remaining,
3996                            deadline_finalization_reserve,
3997                            &prompt,
3998                        ),
3999                    ));
4000                    self.persist_turn_item(&req.thread_id, &turn_id, &item)
4001                        .await?;
4002                    transcript.push(item);
4003                    continue 'tool_rounds;
4004                } else {
4005                    self.start_deadline_finalization(
4006                        &req.thread_id,
4007                        &turn_id,
4008                        &mut transcript,
4009                        remaining,
4010                    )
4011                    .await?;
4012                    deadline_finalization_requested = true;
4013                }
4014            }
4015            if runtime_profile == RuntimeProfile::Eval
4016                && req.task_ledger_required
4017                && !deadline_finalization_requested
4018                && task_ledger_scoreable_checkpoints < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
4019                && let Some(remaining) = deadline_remaining_seconds(turn_deadline)
4020                && remaining <= TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS
4021                && remaining > deadline_finalization_reserve
4022                && let Some(prompt) = task_ledger_completion_prompt(&transcript)
4023            {
4024                task_ledger_scoreable_checkpoints += 1;
4025                let item = TranscriptItem::UserMessage(UserMessage::text(
4026                    task_ledger_scoreable_checkpoint_prompt(remaining, &prompt),
4027                ));
4028                self.persist_turn_item(&req.thread_id, &turn_id, &item)
4029                    .await?;
4030                transcript.push(item);
4031                continue 'tool_rounds;
4032            }
4033            if turn_inference_router.is_active() && routing_candidates.is_none() {
4034                routing_candidates =
4035                    Some(collect_inference_routing_candidates(&self.registry).await);
4036            }
4037            let routing_tools_model = model.clone();
4038            let routing_tools_provider = provider.clone();
4039            let routing_tools = self.filtered_tool_specs(
4040                &cfg,
4041                &model,
4042                &provider,
4043                model_profile.as_ref(),
4044                &thread_overrides.tool_allowlist,
4045                &thread_overrides.external_tools,
4046            );
4047            let prior_failures =
4048                transcript_failure_count_since(&transcript, routing_transcript_start)
4049                    .max(reliability.tool_failure_count())
4050                    .saturating_add(provider_stream_retry_attempts);
4051            let routing_selection = route_inference_selection(
4052                &self.registry,
4053                &turn_inference_router,
4054                InferenceRoutingRequest {
4055                    thread_id: &req.thread_id,
4056                    turn_id: &turn_id,
4057                    round_index: round_index as u32,
4058                    runtime_profile,
4059                    phase: speed_policy.phase(),
4060                    profile: turn_inference_router_profile.as_deref(),
4061                    default_selection: ModelSelection {
4062                        provider: default_provider.clone(),
4063                        model: default_model.clone(),
4064                    },
4065                    transcript: &transcript,
4066                    tools: &routing_tools,
4067                    candidates: routing_candidates.as_deref(),
4068                    prior_failures,
4069                    prior_escalations: routing_escalations,
4070                },
4071            )
4072            .await;
4073            if let Some(decision) = routing_selection.decision.clone() {
4074                if matches!(decision.outcome, InferenceRoutingOutcome::Escalated) {
4075                    routing_escalations = routing_escalations.saturating_add(1);
4076                }
4077                self.emit(RoderEvent::InferenceRoutingDecision(
4078                    InferenceRoutingDecisionEvent {
4079                        thread_id: req.thread_id.clone(),
4080                        turn_id: turn_id.clone(),
4081                        round_index: round_index as u32,
4082                        default_selection: ModelSelection {
4083                            provider: default_provider.clone(),
4084                            model: default_model.clone(),
4085                        },
4086                        selected_selection: routing_selection.selection.clone(),
4087                        decision,
4088                        timestamp: OffsetDateTime::now_utc(),
4089                    },
4090                ))
4091                .await;
4092            }
4093            provider = routing_selection.selection.provider.clone();
4094            model = routing_selection.selection.model.clone();
4095            let engine = self.engine_for(&provider)?;
4096            let capabilities = engine.capabilities();
4097            model_profile = model_profile_for_provider_model(&cfg, &provider, &model);
4098            let tools = if capabilities.tool_calls {
4099                if model == routing_tools_model && provider == routing_tools_provider {
4100                    routing_tools.clone()
4101                } else {
4102                    self.filtered_tool_specs(
4103                        &cfg,
4104                        &model,
4105                        &provider,
4106                        model_profile.as_ref(),
4107                        &thread_overrides.tool_allowlist,
4108                        &thread_overrides.external_tools,
4109                    )
4110                }
4111            } else {
4112                Vec::new()
4113            };
4114            let parallel_tool_calls = parallel_tool_calls_for_model(&cfg, &model);
4115            let tool_choice = if tools.is_empty() {
4116                ToolChoice::None
4117            } else {
4118                ToolChoice::Auto
4119            };
4120            let summary_selection = ModelSelection {
4121                provider: provider.clone(),
4122                model: model.clone(),
4123            };
4124            if model_switch_summary_selection.as_ref() != Some(&summary_selection) {
4125                if let Some(summary) = model_switch_summary(
4126                    &transcript,
4127                    model_profile.as_ref(),
4128                    &provider,
4129                    &model,
4130                    &tools,
4131                ) {
4132                    let item = TranscriptItem::UserMessage(UserMessage::text(summary));
4133                    self.persist_turn_item(&req.thread_id, &turn_id, &item)
4134                        .await?;
4135                    transcript.push(item);
4136                }
4137                model_switch_summary_selection = Some(summary_selection);
4138            }
4139
4140            if !capabilities.image_input && transcript_has_images(&transcript) {
4141                self.fail_turn_with_error(
4142                    &req.thread_id,
4143                    &turn_id,
4144                    format!("provider {provider} does not support image input"),
4145                )
4146                .await?;
4147                return Ok(TurnRunOutcome::Stopped);
4148            }
4149            let compaction_generation_before = self.compaction_generation(&req.thread_id);
4150            transcript = self
4151                .compact_transcript_if_needed(
4152                    &req.thread_id,
4153                    &turn_id,
4154                    &provider,
4155                    &model,
4156                    transcript,
4157                    self.compaction_options_for_turn(&req.thread_id, !compacted_this_turn),
4158                )
4159                .await?;
4160            compacted_this_turn |=
4161                self.compaction_generation(&req.thread_id) != compaction_generation_before;
4162
4163            let speed_policy_decision =
4164                speed_policy.decision(runtime_profile, &model, &cfg.speed_policy);
4165            let request_reasoning = reasoning_from_decision(
4166                speed_policy_decision.as_ref(),
4167                routing_selection
4168                    .reasoning
4169                    .clone()
4170                    .unwrap_or_else(|| reasoning_for_model(&cfg, &model)),
4171            );
4172            self.active_turn_selections.write().await.insert(
4173                turn_id.clone(),
4174                ModelSelectionMode::manual(
4175                    provider.clone(),
4176                    model.clone(),
4177                    request_reasoning.level.clone(),
4178                ),
4179            );
4180            if let Some(limit) = reliability.record_model_call(
4181                &cfg.reliability,
4182                runtime_profile == RuntimeProfile::Interactive,
4183            ) {
4184                self.fail_turn_due_to_reliability_limit(
4185                    &req.thread_id,
4186                    &turn_id,
4187                    &provider,
4188                    &model,
4189                    limit,
4190                    &transcript,
4191                )
4192                .await?;
4193                return Ok(TurnRunOutcome::Stopped);
4194            }
4195            self.emit(RoderEvent::InferenceStarted(InferenceStarted {
4196                thread_id: req.thread_id.clone(),
4197                turn_id: turn_id.clone(),
4198                engine_id: engine.id(),
4199                model: ModelSelection {
4200                    provider: provider.clone(),
4201                    model: model.clone(),
4202                },
4203                reasoning: request_reasoning.clone(),
4204                speed_policy: speed_policy_decision.clone(),
4205                deadline_remaining_seconds: deadline_remaining_seconds(turn_deadline),
4206                timestamp: OffsetDateTime::now_utc(),
4207            }))
4208            .await;
4209
4210            let mut instructions = req.instructions.clone();
4211            if let Some(extra) = &thread_overrides.developer_instructions {
4212                instructions = apply_thread_developer_instructions(instructions, extra);
4213            }
4214            if let Some(context) = req.developer_context.as_deref() {
4215                instructions = apply_turn_developer_context(instructions, context);
4216            }
4217            let mut instructions = apply_runtime_profile(instructions, runtime_profile);
4218            if let Some(profile) = &model_profile {
4219                instructions = apply_model_instruction_overlay(instructions, profile);
4220            }
4221            if req.task_ledger_required
4222                && runtime_profile == RuntimeProfile::Eval
4223                && !transcript_has_task_ledger(&transcript)
4224            {
4225                instructions = apply_task_ledger_required(instructions);
4226            }
4227            if effective_policy_mode == PolicyMode::Plan {
4228                instructions = apply_plan_mode(instructions);
4229            }
4230            if agent_swarm_mode_active {
4231                instructions = apply_agent_swarm_mode(instructions);
4232            }
4233            // Ultra mode (any model) enables proactive multi-agent policy.
4234            // Codex Sol/Terra Ultra effort still does too without requiring the
4235            // mode flag. Models that advertise Ultra effort keep the
4236            // explicit-request-only policy on lower efforts when ultra mode is
4237            // off; other models get multi-agent policy only when ultra mode is on.
4238            let model_has_ultra_effort = model_supports_reasoning_effort(&model, REASONING_ULTRA);
4239            let effort_is_ultra = request_reasoning.level.as_deref() == Some(REASONING_ULTRA);
4240            let proactive_multi_agent = ultra_mode_active || effort_is_ultra;
4241            if ultra_mode_active || model_has_ultra_effort {
4242                instructions = apply_codex_multi_agent_mode(instructions, proactive_multi_agent);
4243            }
4244            instructions = self
4245                .goals
4246                .apply_goal_instructions(&req.thread_id, instructions)
4247                .await?;
4248            instructions =
4249                apply_parallel_web_tools(instructions, tools.iter().map(|spec| spec.name.as_str()));
4250            let mut request_metadata = serde_json::json!({});
4251            if let Some(decision) = &speed_policy_decision {
4252                request_metadata["speedPolicy"] = serde_json::json!(decision);
4253            }
4254            if let Some(decision) = routing_selection.decision.as_ref() {
4255                request_metadata["inferenceRouting"] = serde_json::json!(decision);
4256            }
4257            if let Some(remaining) = deadline_remaining_seconds(turn_deadline) {
4258                request_metadata["deadlineRemainingSeconds"] = serde_json::json!(remaining);
4259            }
4260            if let Some(profile) = &model_profile {
4261                request_metadata["modelProfile"] = serde_json::json!({
4262                    "model": profile.model,
4263                    "providerFamily": profile.provider_family,
4264                    "editTool": profile.edit_tool,
4265                    "schemaPolicy": profile.schema_policy,
4266                    "instructionOverlay": profile.instruction_overlay,
4267                    "parallelToolCalls": profile.parallel_tool_calls,
4268                    "autoCompactTokenLimit": profile.auto_compact_token_limit,
4269                });
4270            }
4271            let task_ledger_required_this_round = req.task_ledger_required
4272                && runtime_profile == RuntimeProfile::Eval
4273                && !deadline_finalization_requested
4274                && !transcript_has_task_ledger(&transcript);
4275            let task_ledger_tools = (capabilities.tool_calls && task_ledger_required_this_round)
4276                .then(|| self.task_ledger_tool_specs(model_profile.as_ref()))
4277                .filter(|tools| !tools.is_empty());
4278            let request_tools = if deadline_finalization_requested {
4279                Vec::new()
4280            } else if let Some(ledger_tools) = &task_ledger_tools {
4281                ledger_tools.clone()
4282            } else {
4283                tools.clone()
4284            };
4285            let request_tool_choice = if deadline_finalization_requested {
4286                ToolChoice::None
4287            } else if task_ledger_tools.is_some() {
4288                ToolChoice::Specific(TASK_LEDGER_TOOL_NAME.to_string())
4289            } else {
4290                tool_choice.clone()
4291            };
4292            if deadline_finalization_requested {
4293                request_metadata["deadlineFinalization"] = serde_json::json!({
4294                    "reserveSeconds": deadline_finalization_reserve,
4295                    "remainingSeconds": deadline_remaining_seconds(turn_deadline),
4296                });
4297            } else if deadline_scoreable_completion_requested {
4298                request_metadata["deadlineScoreableCompletion"] = serde_json::json!({
4299                    "reserveSeconds": deadline_finalization_reserve,
4300                    "remainingSeconds": deadline_remaining_seconds(turn_deadline),
4301                });
4302            }
4303            let mut eager_tools = EagerTools::new(
4304                self,
4305                &request_tools,
4306                &thread_overrides.external_tools,
4307                parallel_tool_calls,
4308            );
4309            let mut request_reliability: ReliabilityRequestPolicy = cfg.reliability.clone().into();
4310            request_reliability.provider_retry_max_attempts = request_reliability
4311                .provider_retry_max_attempts
4312                .saturating_sub(provider_stream_retry_attempts)
4313                .max(1);
4314            let request = AgentInferenceRequest {
4315                model: ModelSelection {
4316                    provider: provider.clone(),
4317                    model: model.clone(),
4318                },
4319                instructions,
4320                transcript: transcript.clone(),
4321                tools: request_tools,
4322                tool_choice: request_tool_choice,
4323                reasoning: request_reasoning,
4324                output: OutputConfig::default(),
4325                runtime: RuntimeHints {
4326                    auto_compact_token_limit: (provider != roder_api::catalog::PROVIDER_CODEX)
4327                        .then(|| server_side_compaction_threshold(&cfg, &model))
4328                        .flatten(),
4329                    profile: runtime_profile,
4330                    parallel_tool_calls: Some(parallel_tool_calls),
4331                    prompt_cache_key: Some(req.thread_id.clone()),
4332                    hosted_web_search: cfg.hosted_web_search.clone(),
4333                    tool_search: tool_search_for_provider_model(&cfg, &provider, &model),
4334                    speed_policy: speed_policy_decision,
4335                    reliability: Some(request_reliability),
4336                    deadline_remaining_seconds: deadline_remaining_seconds(turn_deadline),
4337                    service_tier: req.service_tier_override.clone(),
4338                    ..RuntimeHints::default()
4339                },
4340                metadata: request_metadata,
4341            };
4342
4343            let mut compaction_template = request.clone();
4344            compaction_template.transcript.clear();
4345            self.compaction_templates
4346                .write()
4347                .await
4348                .insert(req.thread_id.clone(), compaction_template);
4349
4350            let ctx = InferenceTurnContext {
4351                thread_id: &req.thread_id,
4352                turn_id: &turn_id,
4353                tool_executor: Some(std::sync::Arc::new(
4354                    crate::tool_execution::RuntimeTurnToolExecutor {
4355                        runtime: Arc::clone(self),
4356                        thread_id: req.thread_id.clone(),
4357                        turn_id: turn_id.clone(),
4358                        workspace: Some(workspace.clone()),
4359                        deadline: turn_deadline,
4360                    },
4361                )),
4362            };
4363            let stream_future = preemptible(engine.stream_turn(ctx, request), &mut steering);
4364            let mut stream = if let Some((deadline, timeout_action)) = inference_timeout_deadline(
4365                turn_deadline,
4366                runtime_profile,
4367                req.task_ledger_required,
4368                deadline_finalization_reserve,
4369                deadline_finalization_requested || deadline_scoreable_completion_requested,
4370                task_ledger_scoreable_checkpoints,
4371                &transcript,
4372            ) {
4373                match tokio::time::timeout_at(deadline_instant(deadline), stream_future).await {
4374                    Ok(stream) => match stream {
4375                        Ok(stream) => stream,
4376                        Err(err) => {
4377                            if err.is::<SamplingPreempted>() {
4378                                self.finish_sampling_cleanup(&turn_id).await?;
4379                                provider_stream_retry_attempts = 0;
4380                                continue 'tool_rounds;
4381                            }
4382                            self.finish_sampling_cleanup(&turn_id).await?;
4383                            if !deadline_finalization_requested
4384                                && self
4385                                    .retry_sampling_failure(
4386                                        SamplingRetry {
4387                                            thread_id: &req.thread_id,
4388                                            turn_id: &turn_id,
4389                                            provider: &provider,
4390                                            model: &model,
4391                                            config: &cfg.reliability,
4392                                            error: &err,
4393                                        },
4394                                        &mut provider_stream_retry_attempts,
4395                                        &mut steering,
4396                                    )
4397                                    .await
4398                            {
4399                                continue 'tool_rounds;
4400                            }
4401                            let error = err.to_string();
4402                            if self
4403                                .try_recover_from_context_limit(
4404                                    &req.thread_id,
4405                                    &turn_id,
4406                                    &provider,
4407                                    &model,
4408                                    &error,
4409                                    &mut transcript,
4410                                    &mut compacted_this_turn,
4411                                    &mut context_limit_recovery_attempts,
4412                                )
4413                                .await?
4414                            {
4415                                continue 'tool_rounds;
4416                            }
4417                            return Err(err);
4418                        }
4419                    },
4420                    Err(_) => {
4421                        if runtime_profile == RuntimeProfile::Eval
4422                            && !deadline_finalization_requested
4423                        {
4424                            let remaining = deadline_remaining_seconds(turn_deadline).unwrap_or(0);
4425                            if timeout_action == InferenceTimeoutAction::ScoreableCheckpoint
4426                                && task_ledger_scoreable_checkpoints
4427                                    < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
4428                                && let Some(prompt) = task_ledger_completion_prompt(&transcript)
4429                            {
4430                                task_ledger_scoreable_checkpoints += 1;
4431                                let item = TranscriptItem::UserMessage(UserMessage::text(
4432                                    task_ledger_scoreable_checkpoint_prompt(remaining, &prompt),
4433                                ));
4434                                self.persist_turn_item(&req.thread_id, &turn_id, &item)
4435                                    .await?;
4436                                transcript.push(item);
4437                                continue 'tool_rounds;
4438                            }
4439                            self.start_deadline_finalization(
4440                                &req.thread_id,
4441                                &turn_id,
4442                                &mut transcript,
4443                                remaining,
4444                            )
4445                            .await?;
4446                            deadline_finalization_requested = true;
4447                            continue 'tool_rounds;
4448                        }
4449                        self.fail_turn_due_to_deadline(
4450                            &req.thread_id,
4451                            &turn_id,
4452                            deadline,
4453                            &transcript,
4454                        )
4455                        .await?;
4456                        return Ok(TurnRunOutcome::Stopped);
4457                    }
4458                }
4459            } else {
4460                match stream_future.await {
4461                    Ok(stream) => stream,
4462                    Err(err) => {
4463                        if err.is::<SamplingPreempted>() {
4464                            self.finish_sampling_cleanup(&turn_id).await?;
4465                            provider_stream_retry_attempts = 0;
4466                            continue 'tool_rounds;
4467                        }
4468                        self.finish_sampling_cleanup(&turn_id).await?;
4469                        if !deadline_finalization_requested
4470                            && self
4471                                .retry_sampling_failure(
4472                                    SamplingRetry {
4473                                        thread_id: &req.thread_id,
4474                                        turn_id: &turn_id,
4475                                        provider: &provider,
4476                                        model: &model,
4477                                        config: &cfg.reliability,
4478                                        error: &err,
4479                                    },
4480                                    &mut provider_stream_retry_attempts,
4481                                    &mut steering,
4482                                )
4483                                .await
4484                        {
4485                            continue 'tool_rounds;
4486                        }
4487                        let error = err.to_string();
4488                        if self
4489                            .try_recover_from_context_limit(
4490                                &req.thread_id,
4491                                &turn_id,
4492                                &provider,
4493                                &model,
4494                                &error,
4495                                &mut transcript,
4496                                &mut compacted_this_turn,
4497                                &mut context_limit_recovery_attempts,
4498                            )
4499                            .await?
4500                        {
4501                            continue 'tool_rounds;
4502                        }
4503                        return Err(err);
4504                    }
4505                }
4506            };
4507            let mut sampling_output = SamplingOutput::default();
4508            let output_context = OutputContext {
4509                thread_id: &req.thread_id,
4510                turn_id: &turn_id,
4511                profile: model_profile.as_ref(),
4512                provider: &provider,
4513                model: &model,
4514            };
4515            let mut assistant_text = String::new();
4516            let mut phase_messages = Vec::<AssistantMessage>::new();
4517            let mut reasoning_text = String::new();
4518            let mut replayed_tool_call = false;
4519            let mut tool_calls = Vec::new();
4520            let mut patch_progress = PatchProgressTracker::default();
4521            let mut provider_metadata = None;
4522            let mut provider_requests_continuation = false;
4523            // Captured from mid-stream Compaction(completed) events. Codex/OpenAI
4524            // sometimes abort the SSE stream after emitting the compaction item
4525            // ("error decoding response body") before response.completed, so we
4526            // must not rely solely on ProviderMetadata for the boundary.
4527            let mut stream_compaction_item: Option<serde_json::Value> = None;
4528
4529            loop {
4530                let next_future = async {
4531                    loop {
4532                        tokio::select! {
4533                            biased;
4534                            _ = wait_for_steer(&mut steering) => break Some(Err(SamplingPreempted.into())),
4535                            result = eager_tools.poll_result(), if eager_tools.pending() => {
4536                                if let Err(error) = result { break Some(Err(error)); }
4537                            }
4538                            next = stream.next() => break next,
4539                        }
4540                    }
4541                };
4542                let next = if let Some((deadline, timeout_action)) = inference_timeout_deadline(
4543                    turn_deadline,
4544                    runtime_profile,
4545                    req.task_ledger_required,
4546                    deadline_finalization_reserve,
4547                    deadline_finalization_requested || deadline_scoreable_completion_requested,
4548                    task_ledger_scoreable_checkpoints,
4549                    &transcript,
4550                ) {
4551                    match tokio::time::timeout_at(deadline_instant(deadline), next_future).await {
4552                        Ok(next) => next,
4553                        Err(_) => {
4554                            if runtime_profile == RuntimeProfile::Eval
4555                                && !deadline_finalization_requested
4556                            {
4557                                let remaining =
4558                                    deadline_remaining_seconds(turn_deadline).unwrap_or(0);
4559                                if timeout_action == InferenceTimeoutAction::ScoreableCheckpoint
4560                                    && task_ledger_scoreable_checkpoints
4561                                        < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
4562                                    && let Some(prompt) = task_ledger_completion_prompt(&transcript)
4563                                {
4564                                    task_ledger_scoreable_checkpoints += 1;
4565                                    let item = TranscriptItem::UserMessage(UserMessage::text(
4566                                        task_ledger_scoreable_checkpoint_prompt(remaining, &prompt),
4567                                    ));
4568                                    self.persist_turn_item(&req.thread_id, &turn_id, &item)
4569                                        .await?;
4570                                    transcript.push(item);
4571                                    continue 'tool_rounds;
4572                                }
4573                                self.start_deadline_finalization(
4574                                    &req.thread_id,
4575                                    &turn_id,
4576                                    &mut transcript,
4577                                    remaining,
4578                                )
4579                                .await?;
4580                                deadline_finalization_requested = true;
4581                                continue 'tool_rounds;
4582                            }
4583                            self.fail_turn_due_to_deadline(
4584                                &req.thread_id,
4585                                &turn_id,
4586                                deadline,
4587                                &transcript,
4588                            )
4589                            .await?;
4590                            return Ok(TurnRunOutcome::Stopped);
4591                        }
4592                    }
4593                } else {
4594                    next_future.await
4595                };
4596                let res = next.unwrap_or_else(|| {
4597                    Err(roder_api::provider_error::ProviderFailure::new(
4598                        roder_api::provider_error::ProviderFailureKind::StreamInterrupted,
4599                        "provider stream ended before terminal completion",
4600                    )
4601                    .into())
4602                });
4603                let event = match res {
4604                    Ok(event) => event,
4605                    Err(err) => {
4606                        drop(stream);
4607                        eager_tools.drain().await?;
4608                        for result in eager_tools.take_results(&tool_calls) {
4609                            verification_gate.record_tool_result(&result);
4610                            transcript.push(TranscriptItem::ToolResult(result));
4611                        }
4612                        self.finish_sampling_cleanup(&turn_id).await?;
4613                        if err.is::<SamplingPreempted>() {
4614                            provider_stream_retry_attempts = 0;
4615                            continue 'tool_rounds;
4616                        }
4617                        if !deadline_finalization_requested
4618                            && self
4619                                .retry_sampling_failure(
4620                                    SamplingRetry {
4621                                        thread_id: &req.thread_id,
4622                                        turn_id: &turn_id,
4623                                        provider: &provider,
4624                                        model: &model,
4625                                        config: &cfg.reliability,
4626                                        error: &err,
4627                                    },
4628                                    &mut provider_stream_retry_attempts,
4629                                    &mut steering,
4630                                )
4631                                .await
4632                        {
4633                            continue 'tool_rounds;
4634                        }
4635                        let error = err.to_string();
4636                        if self
4637                            .try_recover_from_context_limit(
4638                                &req.thread_id,
4639                                &turn_id,
4640                                &provider,
4641                                &model,
4642                                &error,
4643                                &mut transcript,
4644                                &mut compacted_this_turn,
4645                                &mut context_limit_recovery_attempts,
4646                            )
4647                            .await?
4648                        {
4649                            continue 'tool_rounds;
4650                        }
4651                        return Err(err);
4652                    }
4653                };
4654
4655                let inference_timestamp = OffsetDateTime::now_utc();
4656                self.emit(RoderEvent::InferenceEventReceived(InferenceEventReceived {
4657                    thread_id: req.thread_id.clone(),
4658                    turn_id: turn_id.clone(),
4659                    event: event.clone(),
4660                    timestamp: inference_timestamp,
4661                }))
4662                .await;
4663
4664                match event {
4665                    InferenceEvent::MessageDelta(delta) => {
4666                        if let Some((team_id, member)) =
4667                            self.teams.member_for_thread(&req.thread_id).await
4668                        {
4669                            self.emit(RoderEvent::TeamMemberMessageDelta(TeamMemberMessageDelta {
4670                                team_id,
4671                                member_id: member.id,
4672                                member_thread_id: req.thread_id.clone(),
4673                                turn_id: turn_id.clone(),
4674                                delta: delta.text.clone(),
4675                                timestamp: OffsetDateTime::now_utc(),
4676                            }))
4677                            .await;
4678                        }
4679                        if is_final_answer_phase(delta.phase.as_deref()) {
4680                            assistant_text.push_str(&delta.text);
4681                        } else if let Some(last) = phase_messages.last_mut()
4682                            && last.phase == delta.phase
4683                        {
4684                            last.text.push_str(&delta.text);
4685                        } else {
4686                            phase_messages.push(AssistantMessage {
4687                                text: delta.text,
4688                                phase: delta.phase,
4689                            });
4690                        }
4691                    }
4692                    InferenceEvent::ReasoningDelta(delta) => reasoning_text.push_str(&delta.text),
4693                    InferenceEvent::ToolCallCompleted(call) => {
4694                        if completed_call_replayed(&transcript, &call)? {
4695                            replayed_tool_call = true;
4696                            continue;
4697                        }
4698                        if let Some(progress) = patch_progress.complete(
4699                            &req.thread_id,
4700                            &turn_id,
4701                            &call.id,
4702                            &call.name,
4703                            &call.arguments,
4704                        ) {
4705                            self.emit(RoderEvent::PatchProgress(progress)).await;
4706                        }
4707                        if !tool_calls
4708                            .iter()
4709                            .any(|previous: &ToolCallCompleted| previous.id == call.id)
4710                        {
4711                            eager_tools
4712                                .schedule(
4713                                    self,
4714                                    &output_context,
4715                                    &call,
4716                                    workspace.as_str(),
4717                                    turn_deadline,
4718                                    &mut transcript,
4719                                )
4720                                .await?;
4721                            tool_calls.push(call);
4722                        }
4723                    }
4724                    InferenceEvent::OutputItemCompleted(item) => {
4725                        sampling_output
4726                            .complete_item(self, &output_context, item, &mut transcript)
4727                            .await?;
4728                    }
4729                    InferenceEvent::Failed(failure) => {
4730                        speed_policy.record_failure();
4731                        let error = failure.message;
4732                        if self
4733                            .try_recover_from_context_limit(
4734                                &req.thread_id,
4735                                &turn_id,
4736                                &provider,
4737                                &model,
4738                                &error,
4739                                &mut transcript,
4740                                &mut compacted_this_turn,
4741                                &mut context_limit_recovery_attempts,
4742                            )
4743                            .await?
4744                        {
4745                            continue 'tool_rounds;
4746                        }
4747                        self.persist_turn_item(
4748                            &req.thread_id,
4749                            &turn_id,
4750                            &TranscriptItem::Error(ErrorRecord {
4751                                message: error.clone(),
4752                            }),
4753                        )
4754                        .await?;
4755                        self.emit(RoderEvent::TurnFailed(TurnFailed {
4756                            thread_id: req.thread_id.clone(),
4757                            turn_id: turn_id.clone(),
4758                            error: error.clone(),
4759                            error_kind: None,
4760                            usage: None,
4761                            timestamp: OffsetDateTime::now_utc(),
4762                        }))
4763                        .await;
4764                        self.complete_team_member_turn_with_result(
4765                            &req.thread_id,
4766                            &turn_id,
4767                            TeamMemberStatus::Failed,
4768                            (!assistant_text.trim().is_empty()).then(|| assistant_text.clone()),
4769                            Some(error),
4770                        )
4771                        .await?;
4772                        return Ok(TurnRunOutcome::Stopped);
4773                    }
4774                    InferenceEvent::Usage(usage) => {
4775                        turn_usage.add_assign(&usage);
4776                    }
4777                    InferenceEvent::Completed(metadata) => {
4778                        sampling_output
4779                            .finish(
4780                                self,
4781                                &output_context,
4782                                &assistant_text,
4783                                &phase_messages,
4784                                &reasoning_text,
4785                                &mut transcript,
4786                            )
4787                            .await?;
4788                        provider_stream_retry_attempts = 0;
4789                        turn_finish_reason = metadata
4790                            .stop_reason
4791                            .as_deref()
4792                            .map(finish_reason_from_stop_reason);
4793                        break;
4794                    }
4795                    InferenceEvent::Compaction(progress) => {
4796                        // Persist the boundary as soon as the provider emits a
4797                        // completed compaction item. ChatGPT Codex often aborts
4798                        // the SSE stream right after ("error decoding response
4799                        // body"), so waiting for ProviderMetadata loses the
4800                        // boundary and every subsequent request re-compacts.
4801                        if progress.status == "completed"
4802                            && let Some(item) = progress.item
4803                        {
4804                            let already = stream_compaction_item
4805                                .as_ref()
4806                                .and_then(|existing| {
4807                                    existing.get("id").and_then(serde_json::Value::as_str)
4808                                })
4809                                .zip(item.get("id").and_then(serde_json::Value::as_str))
4810                                .is_some_and(|(a, b)| a == b);
4811                            if !already {
4812                                stream_compaction_item = Some(item.clone());
4813                                let boundary = TranscriptItem::ProviderMetadata(
4814                                    crate::compaction::provider_metadata_from_compaction_item(item),
4815                                );
4816                                self.persist_turn_item(&req.thread_id, &turn_id, &boundary)
4817                                    .await?;
4818                                transcript.push(boundary);
4819                                transcript =
4820                                    crate::compaction::trim_to_last_compaction_boundary(transcript);
4821                                compacted_this_turn = true;
4822                            }
4823                        }
4824                    }
4825                    InferenceEvent::HostedToolCallStarted(_)
4826                    | InferenceEvent::HostedToolCallCompleted(_) => {}
4827                    InferenceEvent::ToolCallStarted(call) => {
4828                        patch_progress.start(&call.id, &call.name)
4829                    }
4830                    InferenceEvent::ToolCallDelta(delta) => {
4831                        if let Some(progress) = patch_progress.delta(
4832                            &req.thread_id,
4833                            &turn_id,
4834                            &delta.id,
4835                            &delta.arguments_delta,
4836                        ) {
4837                            self.emit(RoderEvent::PatchProgress(progress)).await;
4838                        }
4839                    }
4840                    InferenceEvent::ProviderMetadata(mut metadata) => {
4841                        if metadata.get("kind").and_then(serde_json::Value::as_str)
4842                            == Some("reliability_retry_attempt")
4843                        {
4844                            provider_stream_retry_attempts =
4845                                provider_stream_retry_attempts.saturating_add(1);
4846                        }
4847                        provider_requests_continuation |= metadata
4848                            .get("end_turn")
4849                            .and_then(serde_json::Value::as_bool)
4850                            == Some(false);
4851                        if let Some(compaction_item) = stream_compaction_item.as_ref() {
4852                            let _ = crate::compaction::ensure_provider_metadata_compaction(
4853                                &mut metadata,
4854                                compaction_item,
4855                            );
4856                        }
4857                        if metadata.get("output").is_none() {
4858                            let item = TranscriptItem::ProviderMetadata(metadata);
4859                            self.persist_turn_item(&req.thread_id, &turn_id, &item)
4860                                .await?;
4861                            transcript.push(item);
4862                        } else {
4863                            provider_metadata = Some(metadata);
4864                        }
4865                    }
4866                }
4867            }
4868
4869            speed_policy.record_model_output(
4870                !assistant_text.is_empty() || !phase_messages.is_empty(),
4871                tool_calls.len(),
4872            );
4873            if tool_calls.is_empty() {
4874                if provider_requests_continuation || replayed_tool_call {
4875                    if let Some(metadata) = provider_metadata {
4876                        let item = TranscriptItem::ProviderMetadata(metadata);
4877                        self.persist_turn_item(&req.thread_id, &turn_id, &item)
4878                            .await?;
4879                        transcript.push(item);
4880                    }
4881                    continue 'tool_rounds;
4882                }
4883                let steers = self.drain_turn_steers(&turn_id, &mut steering).await;
4884                if !steers.is_empty() {
4885                    if let Some(metadata) = provider_metadata {
4886                        let had_provider_compaction =
4887                            crate::compaction::provider_metadata_has_compaction(&metadata);
4888                        let item = TranscriptItem::ProviderMetadata(metadata);
4889                        self.persist_turn_item(&req.thread_id, &turn_id, &item)
4890                            .await?;
4891                        transcript.push(item);
4892                        if had_provider_compaction {
4893                            // Server-side compaction replaced the prior window;
4894                            // drop pre-boundary items so the next request does
4895                            // not re-send (and re-compact) the full history.
4896                            transcript =
4897                                crate::compaction::trim_to_last_compaction_boundary(transcript);
4898                            compacted_this_turn = true;
4899                        }
4900                    }
4901                    self.append_steers(&req, &turn_id, &mut transcript, steers)
4902                        .await?;
4903                    continue;
4904                }
4905                if !deadline_finalization_requested
4906                    && req.task_ledger_required
4907                    && runtime_profile == RuntimeProfile::Eval
4908                    && task_ledger_completion_reminders < TASK_LEDGER_COMPLETION_REMINDER_LIMIT
4909                    && (!assistant_text.trim().is_empty() || !phase_messages.is_empty())
4910                    && let Some(prompt) = task_ledger_completion_prompt(&transcript)
4911                {
4912                    task_ledger_completion_reminders += 1;
4913                    let item = TranscriptItem::UserMessage(UserMessage::text(prompt));
4914                    self.persist_turn_item(&req.thread_id, &turn_id, &item)
4915                        .await?;
4916                    transcript.push(item);
4917                    continue;
4918                }
4919                if !deadline_finalization_requested
4920                    && let Some(prompt) = verification_gate.blocking_prompt()
4921                {
4922                    speed_policy.record_verification_required();
4923                    self.emit(RoderEvent::VerificationRequired(VerificationRequired {
4924                        thread_id: req.thread_id.clone(),
4925                        turn_id: turn_id.clone(),
4926                        reason: verification_gate.reason(),
4927                        changed_files: verification_gate.changed_files(),
4928                        tool_evidence: verification_gate.tool_evidence.clone(),
4929                        tests_run: verification_gate.tests_run.clone(),
4930                        open_gaps: verification_gate.open_gaps.clone(),
4931                        timestamp: OffsetDateTime::now_utc(),
4932                    }))
4933                    .await;
4934                    let item = TranscriptItem::UserMessage(UserMessage::text(prompt));
4935                    self.persist_turn_item(&req.thread_id, &turn_id, &item)
4936                        .await?;
4937                    transcript.push(item);
4938                    continue;
4939                }
4940                // Loop persistence nudge: in non-interactive/eval runs, when the
4941                // model returns a final message with no tool calls, gently prompt
4942                // it to keep going (bounded by `empty_tool_call_nudges`) before
4943                // ending the turn. Gated to non-interactive/eval so interactive
4944                // chat completions are never delayed, and only fires when the
4945                // model actually produced a final answer to re-examine.
4946                if !deadline_finalization_requested
4947                    && runtime_profile != RuntimeProfile::Interactive
4948                    && cfg.reliability.empty_tool_call_nudges > 0
4949                    && empty_tool_call_nudges_used < cfg.reliability.empty_tool_call_nudges
4950                    && (!assistant_text.trim().is_empty() || !phase_messages.is_empty())
4951                {
4952                    empty_tool_call_nudges_used += 1;
4953                    let item = TranscriptItem::UserMessage(UserMessage::text(
4954                        RELIABILITY_CONTINUATION_PROMPT.to_string(),
4955                    ));
4956                    self.persist_turn_item(&req.thread_id, &turn_id, &item)
4957                        .await?;
4958                    transcript.push(item);
4959                    continue;
4960                }
4961                if deadline_finalization_requested
4962                    && assistant_text.trim().is_empty()
4963                    && phase_messages.is_empty()
4964                {
4965                    assistant_text = format!(
4966                        "Deadline finalization completed without model text. {}",
4967                        turn_partial_result(&transcript)
4968                    );
4969                }
4970                final_assistant_text = assistant_text;
4971                final_provider_metadata = provider_metadata;
4972                exhausted_tool_rounds = false;
4973                break;
4974            }
4975
4976            if let Some(metadata) = provider_metadata {
4977                let had_provider_compaction =
4978                    crate::compaction::provider_metadata_has_compaction(&metadata);
4979                let item = TranscriptItem::ProviderMetadata(metadata);
4980                self.persist_turn_item(&req.thread_id, &turn_id, &item)
4981                    .await?;
4982                transcript.push(item);
4983                if had_provider_compaction {
4984                    // Server-side compaction replaced the prior window; drop
4985                    // pre-boundary items so subsequent tool rounds use the
4986                    // compact window as the next input.
4987                    transcript = crate::compaction::trim_to_last_compaction_boundary(transcript);
4988                    compacted_this_turn = true;
4989                }
4990            }
4991            eager_tools.drain().await?;
4992            for call in &tool_calls {
4993                if eager_tools.scheduled(&call.id) {
4994                    continue;
4995                }
4996                let tool_item = TranscriptItem::ToolCall(ToolCallRecord {
4997                    id: call.id.clone(),
4998                    name: call.name.clone(),
4999                    arguments: call.arguments.clone(),
5000                });
5001                self.persist_turn_item(&req.thread_id, &turn_id, &tool_item)
5002                    .await?;
5003                transcript.push(tool_item);
5004                self.persist_model_profile_segment(
5005                    &req.thread_id,
5006                    &turn_id,
5007                    model_profile.as_ref(),
5008                    &provider,
5009                    &model,
5010                    "tool_call",
5011                )
5012                .await?;
5013            }
5014            if let Some(deadline) = turn_deadline
5015                && deadline_expired(deadline)
5016            {
5017                self.fail_turn_due_to_deadline(&req.thread_id, &turn_id, deadline, &transcript)
5018                    .await?;
5019                return Ok(TurnRunOutcome::Stopped);
5020            }
5021            let mut results = eager_tools.take_results(&tool_calls);
5022            let remaining_calls = tool_calls
5023                .into_iter()
5024                .filter(|call| !eager_tools.scheduled(&call.id))
5025                .collect();
5026            results.extend(
5027                self.route_tool_calls(
5028                    &req.thread_id,
5029                    &turn_id,
5030                    remaining_calls,
5031                    parallel_tool_calls,
5032                    Some(workspace.as_str()),
5033                    turn_deadline,
5034                )
5035                .await?,
5036            );
5037            let reliability_limit = reliability.record_tool_results(
5038                &cfg.reliability,
5039                &results,
5040                runtime_profile == RuntimeProfile::Interactive,
5041            );
5042            for result in results {
5043                verification_gate.record_tool_result(&result);
5044                transcript.push(TranscriptItem::ToolResult(result));
5045                self.persist_model_profile_segment(
5046                    &req.thread_id,
5047                    &turn_id,
5048                    model_profile.as_ref(),
5049                    &provider,
5050                    &model,
5051                    "tool_result",
5052                )
5053                .await?;
5054            }
5055            if let Some(limit) = reliability_limit {
5056                if limit.decision == ReliabilityLimitDecision::RequestContinuation {
5057                    // Loop persistence: rather than ending the turn, reset the
5058                    // consecutive-failure counter, nudge the model to keep going,
5059                    // and continue the round loop. Bounded by the per-turn tool
5060                    // failure ceiling, `max_model_calls_per_turn`, and
5061                    // `MAX_TOOL_ROUNDS_PER_TURN`.
5062                    self.record_reliability_limit_continuation(
5063                        &req.thread_id,
5064                        &turn_id,
5065                        &provider,
5066                        &model,
5067                        limit,
5068                    )
5069                    .await;
5070                    reliability.reset_consecutive_failures();
5071                    let nudge = TranscriptItem::UserMessage(UserMessage::text(
5072                        RELIABILITY_CONTINUATION_PROMPT.to_string(),
5073                    ));
5074                    self.persist_turn_item(&req.thread_id, &turn_id, &nudge)
5075                        .await?;
5076                    transcript.push(nudge);
5077                } else {
5078                    self.fail_turn_due_to_reliability_limit(
5079                        &req.thread_id,
5080                        &turn_id,
5081                        &provider,
5082                        &model,
5083                        limit,
5084                        &transcript,
5085                    )
5086                    .await?;
5087                    return Ok(TurnRunOutcome::Stopped);
5088                }
5089            }
5090            let compaction_generation_before = self.compaction_generation(&req.thread_id);
5091            transcript = self
5092                .compact_transcript_if_needed(
5093                    &req.thread_id,
5094                    &turn_id,
5095                    &provider,
5096                    &model,
5097                    transcript,
5098                    self.compaction_options_for_turn(&req.thread_id, !compacted_this_turn),
5099                )
5100                .await?;
5101            compacted_this_turn |=
5102                self.compaction_generation(&req.thread_id) != compaction_generation_before;
5103        }
5104
5105        if exhausted_tool_rounds {
5106            let message =
5107                format!("tool call limit reached after {MAX_TOOL_ROUNDS_PER_TURN} rounds");
5108            self.persist_turn_item(
5109                &req.thread_id,
5110                &turn_id,
5111                &TranscriptItem::Error(ErrorRecord {
5112                    message: message.clone(),
5113                }),
5114            )
5115            .await?;
5116            self.emit(RoderEvent::TurnFailed(TurnFailed {
5117                thread_id: req.thread_id.clone(),
5118                turn_id: turn_id.clone(),
5119                error: message.clone(),
5120                error_kind: None,
5121                usage: None,
5122                timestamp: OffsetDateTime::now_utc(),
5123            }))
5124            .await;
5125            self.complete_team_member_turn_with_result(
5126                &req.thread_id,
5127                &turn_id,
5128                TeamMemberStatus::Failed,
5129                None,
5130                Some(message),
5131            )
5132            .await?;
5133            return Ok(TurnRunOutcome::Stopped);
5134        }
5135
5136        if let Some(metadata) = final_provider_metadata {
5137            self.persist_turn_item(
5138                &req.thread_id,
5139                &turn_id,
5140                &TranscriptItem::ProviderMetadata(metadata),
5141            )
5142            .await?;
5143        }
5144
5145        let turn_usage_tokens = turn_usage.total_tokens as i64;
5146        let completed_usage = (!turn_usage.is_empty()).then_some(turn_usage.clone());
5147        self.record_thread_usage_metadata(&req.thread_id, &turn_usage)
5148            .await?;
5149        self.goals
5150            .account_turn_usage(
5151                &req.thread_id,
5152                turn_usage_tokens,
5153                OffsetDateTime::now_utc() - turn_started_at,
5154            )
5155            .await?;
5156        let (cleanup, ownership) = self.await_provider_turn_cleanup(&turn_id).await;
5157        self.record_turn_lifecycle_with_ownership(
5158            req.thread_id.clone(),
5159            turn_id.clone(),
5160            TurnLifecycleState::Completed,
5161            cleanup,
5162            None,
5163            ownership,
5164        )
5165        .await;
5166        self.emit(RoderEvent::TurnCompleted(TurnCompleted {
5167            thread_id: req.thread_id.clone(),
5168            turn_id: turn_id.clone(),
5169            usage: completed_usage,
5170            finish_reason: turn_finish_reason,
5171            timestamp: OffsetDateTime::now_utc(),
5172        }))
5173        .await;
5174        self.complete_team_member_turn_with_result(
5175            &req.thread_id,
5176            &turn_id,
5177            TeamMemberStatus::Completed,
5178            (!final_assistant_text.is_empty()).then_some(final_assistant_text),
5179            None,
5180        )
5181        .await?;
5182        Ok(TurnRunOutcome::Completed)
5183    }
5184
5185    async fn route_tool_calls(
5186        self: &Arc<Self>,
5187        thread_id: &ThreadId,
5188        turn_id: &TurnId,
5189        calls: Vec<ToolCallCompleted>,
5190        parallel: bool,
5191        workspace: Option<&str>,
5192        deadline: Option<OffsetDateTime>,
5193    ) -> anyhow::Result<Vec<ToolResultRecord>> {
5194        // Enforce the agent_swarm exclusivity rule (roadmap 104, Task 2):
5195        // `agent_swarm` must be the only tool call in a model response. A mixed
5196        // or multi-swarm batch is denied wholesale with actionable retry text so
5197        // the model re-issues `agent_swarm` by itself, and every tool_call_id
5198        // still gets a response (keeping the chat-completions transcript valid).
5199        if let Some(violation) = roder_api::subagents::agent_swarm_batch_violation(
5200            calls.iter().map(|call| call.name.as_str()),
5201        ) {
5202            let message = violation.deny_message();
5203            return Ok(calls
5204                .into_iter()
5205                .map(|call| ToolResultRecord {
5206                    id: call.id,
5207                    name: Some(call.name),
5208                    result: message.clone(),
5209                    display_payload: None,
5210                    is_error: true,
5211                })
5212                .collect());
5213        }
5214        // Emit swarm lifecycle events on the bus (roadmap 104, Task 1) so any
5215        // app-server/SDK/TUI client can observe a swarm as a whole; per-child
5216        // progress flows through the existing Subagent* trace events.
5217        let swarm_call = (calls.len() == 1
5218            && calls[0].name == roder_api::subagents::AGENT_SWARM_TOOL_NAME)
5219            .then(|| (calls[0].id.clone(), calls[0].arguments.clone()));
5220        if let Some((tool_id, args)) = &swarm_call {
5221            self.emit(RoderEvent::AgentSwarmStarted(
5222                roder_api::subagents::AgentSwarmStarted {
5223                    thread_id: thread_id.clone(),
5224                    turn_id: turn_id.clone(),
5225                    tool_id: tool_id.clone(),
5226                    child_count: agent_swarm_child_count(args),
5227                    timestamp: OffsetDateTime::now_utc(),
5228                },
5229            ))
5230            .await;
5231        }
5232
5233        let force_sequential = calls
5234            .iter()
5235            .any(|call| crate::agent_control_tools::is_agent_control_tool(&call.name));
5236        let results =
5237            if parallel && !force_sequential {
5238                try_join_all(calls.into_iter().map(|call| {
5239                    self.route_tool_call(thread_id, turn_id, call, workspace, deadline)
5240                }))
5241                .await
5242            } else {
5243                let mut results = Vec::with_capacity(calls.len());
5244                for call in calls {
5245                    results.push(
5246                        self.route_tool_call(thread_id, turn_id, call, workspace, deadline)
5247                            .await?,
5248                    );
5249                }
5250                Ok(results)
5251            }?;
5252
5253        if let Some((tool_id, _)) = &swarm_call
5254            && let Some(result) = results.iter().find(|result| &result.id == tool_id)
5255            && let Some((completed, failed, aborted)) = parse_swarm_counts(&result.result)
5256        {
5257            self.emit(RoderEvent::AgentSwarmCompleted(
5258                roder_api::subagents::AgentSwarmCompleted {
5259                    thread_id: thread_id.clone(),
5260                    turn_id: turn_id.clone(),
5261                    tool_id: tool_id.clone(),
5262                    completed,
5263                    failed,
5264                    aborted,
5265                    timestamp: OffsetDateTime::now_utc(),
5266                },
5267            ))
5268            .await;
5269        }
5270
5271        Ok(results)
5272    }
5273
5274    /// On provider context/prompt-length failures, force-compact (and if needed
5275    /// strip the last bulky item) then retry the current tool round instead of
5276    /// failing the turn. Bounded so a permanently oversized suffix still stops.
5277    async fn try_recover_from_context_limit(
5278        &self,
5279        thread_id: &ThreadId,
5280        turn_id: &TurnId,
5281        provider: &str,
5282        model: &str,
5283        error: &str,
5284        transcript: &mut Vec<TranscriptItem>,
5285        compacted_this_turn: &mut bool,
5286        recovery_attempts: &mut u32,
5287    ) -> anyhow::Result<bool> {
5288        const MAX_CONTEXT_LIMIT_RECOVERY_ATTEMPTS: u32 = 2;
5289        if !crate::compaction::is_context_limit_failure_message(error) {
5290            return Ok(false);
5291        }
5292        if *recovery_attempts >= MAX_CONTEXT_LIMIT_RECOVERY_ATTEMPTS {
5293            return Ok(false);
5294        }
5295        *recovery_attempts = recovery_attempts.saturating_add(1);
5296
5297        let error_item = TranscriptItem::Error(ErrorRecord {
5298            message: error.to_string(),
5299        });
5300        self.persist_turn_item(thread_id, turn_id, &error_item)
5301            .await?;
5302        transcript.push(error_item);
5303
5304        // After the first failed recovery, drop the item ahead of the latest
5305        // user/error so a second compact can succeed when the suffix itself is
5306        // still oversized (e.g. a huge tool result right before the prompt).
5307        if *recovery_attempts > 1 {
5308            *transcript =
5309                crate::compaction::strip_last_message_before_error(std::mem::take(transcript));
5310        }
5311
5312        let force_options = crate::compaction::CompactionOptions {
5313            allow_repeat: true,
5314            force: true,
5315            hysteresis_baseline: None,
5316            preserve_hint: Some(
5317                "Recover from a provider context/prompt-length limit; preserve the active user goal and recent tool outcomes."
5318                    .to_string(),
5319            ),
5320        };
5321        let before_len = transcript.len();
5322        let before_tokens = crate::compaction::estimate_prompt_tokens(transcript);
5323        *transcript = self
5324            .compact_transcript_if_needed(
5325                thread_id,
5326                turn_id,
5327                provider,
5328                model,
5329                std::mem::take(transcript),
5330                force_options,
5331            )
5332            .await?;
5333        let after_tokens = crate::compaction::estimate_prompt_tokens(transcript);
5334        *compacted_this_turn = *compacted_this_turn
5335            || transcript
5336                .iter()
5337                .any(crate::compaction::is_compaction_boundary);
5338
5339        self.emit(RoderEvent::ReliabilityRetryRecorded(
5340            ReliabilityRetryRecorded {
5341                context: ReliabilityContext {
5342                    thread_id: thread_id.clone(),
5343                    turn_id: turn_id.clone(),
5344                    provider: Some(provider.to_string()),
5345                    model: Some(model.to_string()),
5346                    ..ReliabilityContext::default()
5347                },
5348                error_class: ReliabilityErrorClass::ProviderError,
5349                decision: ReliabilityRetryDecision::Retry,
5350                attempt: *recovery_attempts,
5351                max_attempts: MAX_CONTEXT_LIMIT_RECOVERY_ATTEMPTS,
5352                delay_ms: Some(0),
5353                details: ReliabilityDetails::redacted(format!(
5354                    "context_limit_recovery: compact {before_tokens}->{after_tokens} tokens (items {before_len}->{}); {error}",
5355                    transcript.len()
5356                )),
5357                timestamp: OffsetDateTime::now_utc(),
5358            },
5359        ))
5360        .await;
5361
5362        // If force-compact did not shrink anything and we still have budget,
5363        // strip once more so the next round is not identical.
5364        if after_tokens >= before_tokens && *recovery_attempts < MAX_CONTEXT_LIMIT_RECOVERY_ATTEMPTS
5365        {
5366            *transcript =
5367                crate::compaction::strip_last_message_before_error(std::mem::take(transcript));
5368        }
5369        Ok(true)
5370    }
5371
5372    async fn fail_turn_with_error(
5373        &self,
5374        thread_id: &ThreadId,
5375        turn_id: &TurnId,
5376        message: String,
5377    ) -> anyhow::Result<()> {
5378        self.persist_turn_item(
5379            thread_id,
5380            turn_id,
5381            &TranscriptItem::Error(ErrorRecord {
5382                message: message.clone(),
5383            }),
5384        )
5385        .await?;
5386        self.emit(RoderEvent::TurnFailed(TurnFailed {
5387            thread_id: thread_id.clone(),
5388            turn_id: turn_id.clone(),
5389            error: message.clone(),
5390            error_kind: None,
5391            usage: None,
5392            timestamp: OffsetDateTime::now_utc(),
5393        }))
5394        .await;
5395        self.complete_team_member_turn_with_result(
5396            thread_id,
5397            turn_id,
5398            TeamMemberStatus::Failed,
5399            None,
5400            Some(message),
5401        )
5402        .await?;
5403        Ok(())
5404    }
5405
5406    async fn fail_turn_due_to_deadline(
5407        &self,
5408        thread_id: &ThreadId,
5409        turn_id: &TurnId,
5410        deadline: OffsetDateTime,
5411        transcript: &[TranscriptItem],
5412    ) -> anyhow::Result<()> {
5413        let partial_result = turn_partial_result(transcript);
5414        self.emit(RoderEvent::TurnPartialResult(TurnPartialResult {
5415            thread_id: thread_id.clone(),
5416            turn_id: turn_id.clone(),
5417            summary: partial_result.clone(),
5418            timestamp: OffsetDateTime::now_utc(),
5419        }))
5420        .await;
5421        self.emit(RoderEvent::TurnDeadlineExceeded(TurnDeadlineExceeded {
5422            thread_id: thread_id.clone(),
5423            turn_id: turn_id.clone(),
5424            deadline,
5425            partial_result: partial_result.clone(),
5426            timestamp: OffsetDateTime::now_utc(),
5427        }))
5428        .await;
5429        let message = "turn deadline expired".to_string();
5430        self.persist_turn_item(
5431            thread_id,
5432            turn_id,
5433            &TranscriptItem::Error(ErrorRecord {
5434                message: format!("{message}: {partial_result}"),
5435            }),
5436        )
5437        .await?;
5438        self.emit(RoderEvent::TurnFailed(TurnFailed {
5439            thread_id: thread_id.clone(),
5440            turn_id: turn_id.clone(),
5441            error: message.clone(),
5442            error_kind: Some("deadline_timeout".to_string()),
5443            usage: None,
5444            timestamp: OffsetDateTime::now_utc(),
5445        }))
5446        .await;
5447        self.complete_team_member_turn_with_result(
5448            thread_id,
5449            turn_id,
5450            TeamMemberStatus::Failed,
5451            None,
5452            Some(format!("{message}: {partial_result}")),
5453        )
5454        .await?;
5455        Ok(())
5456    }
5457
5458    async fn start_deadline_finalization(
5459        &self,
5460        thread_id: &ThreadId,
5461        turn_id: &TurnId,
5462        transcript: &mut Vec<TranscriptItem>,
5463        remaining_seconds: u64,
5464    ) -> anyhow::Result<()> {
5465        let item = TranscriptItem::UserMessage(crate::deadline_policy::finalization_message(
5466            remaining_seconds,
5467        ));
5468        self.persist_turn_item(thread_id, turn_id, &item).await?;
5469        transcript.push(item);
5470        self.emit(RoderEvent::TurnPartialResult(TurnPartialResult {
5471            thread_id: thread_id.clone(),
5472            turn_id: turn_id.clone(),
5473            summary: turn_partial_result(transcript),
5474            timestamp: OffsetDateTime::now_utc(),
5475        }))
5476        .await;
5477        Ok(())
5478    }
5479
5480    /// Emits the reliability-limit event for a continuation (loop persistence)
5481    /// without failing the turn. The caller resets the failure counter and
5482    /// injects a nudge so the round loop keeps going.
5483    async fn record_reliability_limit_continuation(
5484        &self,
5485        thread_id: &ThreadId,
5486        turn_id: &TurnId,
5487        provider: &str,
5488        model: &str,
5489        limit: ReliabilityLimitHit,
5490    ) {
5491        self.emit(RoderEvent::ReliabilityLimitRecorded(
5492            ReliabilityLimitRecorded {
5493                context: ReliabilityContext {
5494                    thread_id: thread_id.clone(),
5495                    turn_id: turn_id.clone(),
5496                    tool_id: None,
5497                    tool_name: None,
5498                    provider: Some(provider.to_string()),
5499                    model: Some(model.to_string()),
5500                },
5501                error_class: limit.error_class,
5502                limit_kind: limit.limit_kind,
5503                decision: limit.decision,
5504                current: limit.current,
5505                limit: limit.limit,
5506                details: ReliabilityDetails::redacted(&limit.message),
5507                timestamp: OffsetDateTime::now_utc(),
5508            },
5509        ))
5510        .await;
5511    }
5512
5513    async fn fail_turn_due_to_reliability_limit(
5514        &self,
5515        thread_id: &ThreadId,
5516        turn_id: &TurnId,
5517        provider: &str,
5518        model: &str,
5519        limit: ReliabilityLimitHit,
5520        transcript: &[TranscriptItem],
5521    ) -> anyhow::Result<()> {
5522        self.emit(RoderEvent::ReliabilityLimitRecorded(
5523            ReliabilityLimitRecorded {
5524                context: ReliabilityContext {
5525                    thread_id: thread_id.clone(),
5526                    turn_id: turn_id.clone(),
5527                    tool_id: None,
5528                    tool_name: None,
5529                    provider: Some(provider.to_string()),
5530                    model: Some(model.to_string()),
5531                },
5532                error_class: limit.error_class,
5533                limit_kind: limit.limit_kind,
5534                decision: limit.decision,
5535                current: limit.current,
5536                limit: limit.limit,
5537                details: ReliabilityDetails::redacted(&limit.message),
5538                timestamp: OffsetDateTime::now_utc(),
5539            },
5540        ))
5541        .await;
5542        let partial_result = turn_partial_result(transcript);
5543        self.emit(RoderEvent::TurnPartialResult(TurnPartialResult {
5544            thread_id: thread_id.clone(),
5545            turn_id: turn_id.clone(),
5546            summary: partial_result.clone(),
5547            timestamp: OffsetDateTime::now_utc(),
5548        }))
5549        .await;
5550        let message = format!("reliability limit reached: {}", limit.message);
5551        self.persist_turn_item(
5552            thread_id,
5553            turn_id,
5554            &TranscriptItem::Error(ErrorRecord {
5555                message: format!("{message}: {partial_result}"),
5556            }),
5557        )
5558        .await?;
5559        self.emit(RoderEvent::TurnFailed(TurnFailed {
5560            thread_id: thread_id.clone(),
5561            turn_id: turn_id.clone(),
5562            error: message.clone(),
5563            error_kind: Some("reliability_limit".to_string()),
5564            usage: None,
5565            timestamp: OffsetDateTime::now_utc(),
5566        }))
5567        .await;
5568        self.complete_team_member_turn_with_result(
5569            thread_id,
5570            turn_id,
5571            TeamMemberStatus::Failed,
5572            None,
5573            Some(format!("{message}: {partial_result}")),
5574        )
5575        .await?;
5576        Ok(())
5577    }
5578
5579    async fn append_steers(
5580        &self,
5581        req: &StartTurnRequest,
5582        turn_id: &TurnId,
5583        transcript: &mut Vec<TranscriptItem>,
5584        steers: Vec<QueuedTurnSteer>,
5585    ) -> anyhow::Result<()> {
5586        for queued in steers {
5587            let mut steer = queued.message;
5588            steer.text = steer.text.trim().to_string();
5589            if steer.text.is_empty() && steer.images.is_empty() {
5590                continue;
5591            }
5592            let item = TranscriptItem::UserMessage(steer);
5593            self.persist_turn_item(&req.thread_id, turn_id, &item)
5594                .await?;
5595            transcript.push(item);
5596            if let Some(ack) = queued.mailbox_ack {
5597                self.teams
5598                    .mark_mailbox_messages_delivered(&ack.team_id, turn_id, &ack.message_ids)
5599                    .await?;
5600            }
5601        }
5602        Ok(())
5603    }
5604
5605    async fn persist_model_profile_segment(
5606        &self,
5607        thread_id: &ThreadId,
5608        turn_id: &TurnId,
5609        profile: Option<&ModelHarnessProfile>,
5610        provider: &str,
5611        model: &str,
5612        segment: &str,
5613    ) -> anyhow::Result<()> {
5614        let item = TranscriptItem::ProviderMetadata(model_profile_segment_metadata(
5615            profile, provider, model, segment,
5616        ));
5617        self.persist_turn_item(thread_id, turn_id, &item).await
5618    }
5619
5620    fn task_ledger_tool_specs(
5621        &self,
5622        profile: Option<&ModelHarnessProfile>,
5623    ) -> Vec<roder_api::tools::ToolSpec> {
5624        self.tool_registry
5625            .get(TASK_LEDGER_TOOL_NAME)
5626            .map(|tool| {
5627                tool.spec()
5628                    .normalized_for_model_profile(schema_policy_for_model(profile))
5629            })
5630            .into_iter()
5631            .collect()
5632    }
5633
5634    pub(crate) fn engine_for(&self, provider: &str) -> anyhow::Result<Arc<dyn InferenceEngine>> {
5635        self.registry
5636            .inference_engine(provider)
5637            .or_else(|| {
5638                self.registry
5639                    .default_inference_engine()
5640                    .filter(|engine| provider.is_empty() || engine.id() == provider)
5641            })
5642            .ok_or_else(|| anyhow::anyhow!("inference provider {provider:?} is not registered"))
5643    }
5644
5645    pub async fn emit(&self, event: RoderEvent) -> EventEnvelope {
5646        self.dispatch_local_hook_lifecycle(&event).await;
5647        if let Some(record) = Self::lifecycle_record_for_event(&event) {
5648            let _ = self.persist_turn_lifecycle_record(&record).await;
5649        }
5650        let envelope = self.bus.emit(event);
5651        if let (Some(store), Some(thread_id)) = (&self.thread_store, envelope.thread_id.as_ref())
5652            && should_persist_thread_event(thread_id)
5653        {
5654            let _ = store.append_event(thread_id, &envelope).await;
5655        }
5656        // Registered event sinks (e.g. process extensions) receive the
5657        // persisted envelope through bounded per-sink queues; a slow sink
5658        // never blocks emit or turn progress.
5659        let dispatcher = self
5660            .event_sink_dispatcher
5661            .get_or_init(|| async {
5662                crate::event_sink_dispatch::EventSinkDispatcher::start(
5663                    &self.registry.event_sinks,
5664                    self.bus.clone(),
5665                )
5666            })
5667            .await;
5668        if !dispatcher.is_empty() {
5669            dispatcher.dispatch(&envelope, &self.bus);
5670        }
5671        envelope
5672    }
5673
5674    async fn dispatch_local_hook_lifecycle(&self, event: &RoderEvent) {
5675        // SessionStart belongs to the session, not the turn: firing it from
5676        // TurnStarted re-ran setup hooks on every user message. ThreadCreated
5677        // and ThreadLoaded carry no turn id, so they borrow a synthetic one the
5678        // way the SessionEnd path does.
5679        let session_turn_id = SESSION_HOOK_TURN_ID.to_string();
5680        let (thread_id, turn_id, name, matcher, input) = match event {
5681            RoderEvent::ThreadCreated(event) => (
5682                &event.thread_id,
5683                &session_turn_id,
5684                "SessionStart",
5685                Some("startup"),
5686                serde_json::json!({"source":"startup"}),
5687            ),
5688            RoderEvent::ThreadLoaded(event) => (
5689                &event.thread_id,
5690                &session_turn_id,
5691                "SessionStart",
5692                Some("resume"),
5693                serde_json::json!({"source":"resume"}),
5694            ),
5695            RoderEvent::TurnCompleted(event) => (
5696                &event.thread_id,
5697                &event.turn_id,
5698                "Stop",
5699                None,
5700                serde_json::json!({"finishReason":event.finish_reason}),
5701            ),
5702            RoderEvent::TurnFailed(event) => (
5703                &event.thread_id,
5704                &event.turn_id,
5705                "Stop",
5706                None,
5707                serde_json::json!({"error":event.error}),
5708            ),
5709            RoderEvent::ContextCompactionStarted(event) => (
5710                &event.thread_id,
5711                &event.turn_id,
5712                "PreCompact",
5713                None,
5714                serde_json::json!({"originalItemCount":event.original_item_count,"originalEstimatedTokens":event.original_estimated_tokens}),
5715            ),
5716            RoderEvent::ContextCompactionRecorded(event) => (
5717                &event.thread_id,
5718                &event.turn_id,
5719                "PostCompact",
5720                None,
5721                serde_json::json!({"compactedItemCount":event.compacted_item_count,"compactedEstimatedTokens":event.compacted_estimated_tokens}),
5722            ),
5723            RoderEvent::SubagentStarted(event) => (
5724                &event.parent_thread_id,
5725                &event.parent_turn_id,
5726                "SubagentStart",
5727                Some(event.agent_type.as_str()),
5728                serde_json::json!({"subagentThreadId":event.thread_id,"agentType":event.agent_type,"description":event.description}),
5729            ),
5730            RoderEvent::SubagentCompleted(event) => (
5731                &event.parent_thread_id,
5732                &event.parent_turn_id,
5733                "SubagentStop",
5734                Some(event.agent_type.as_str()),
5735                serde_json::json!({"subagentThreadId":event.thread_id,"agentType":event.agent_type,"exitReason":event.exit_reason}),
5736            ),
5737            RoderEvent::SubagentFailed(event) => (
5738                &event.parent_thread_id,
5739                &event.parent_turn_id,
5740                "SubagentStop",
5741                Some(event.agent_type.as_str()),
5742                serde_json::json!({"subagentThreadId":event.thread_id,"agentType":event.agent_type,"error":event.error}),
5743            ),
5744            _ => return,
5745        };
5746        // One SessionStart per session, whichever event opened it.
5747        if name == "SessionStart" && !self.claim_session_hook_start(thread_id).await {
5748            return;
5749        }
5750        let workspace = self.config.read().await.workspace.clone();
5751        crate::hooks::run_lifecycle(
5752            self,
5753            thread_id,
5754            turn_id,
5755            workspace.as_deref(),
5756            name,
5757            matcher,
5758            input,
5759        )
5760        .await;
5761    }
5762
5763    /// Claim the single `SessionStart` dispatch for a thread, returning whether
5764    /// this caller won it.
5765    pub(crate) async fn claim_session_hook_start(&self, thread_id: &ThreadId) -> bool {
5766        self.session_hook_started
5767            .write()
5768            .await
5769            .insert(thread_id.clone())
5770    }
5771
5772    /// Records a projected thread item event and persists it through the configured thread store.
5773    ///
5774    /// App-server protocol notification bridges currently call this after translating runtime
5775    /// events into item events. Headless runtime consumers that subscribe to `RoderEvent` directly
5776    /// must make the same call if they need the derived item event stream persisted.
5777    pub async fn record_thread_item_event_kind(
5778        &self,
5779        thread_id: &ThreadId,
5780        turn_id: &TurnId,
5781        timestamp: OffsetDateTime,
5782        kind: ThreadItemEventKind,
5783    ) -> anyhow::Result<ThreadItemEvent> {
5784        let seq = self.next_thread_item_event_seq(thread_id).await?;
5785        let item_event = ThreadItemEvent {
5786            seq,
5787            event_id: uuid::Uuid::new_v4().to_string(),
5788            thread_id: thread_id.clone(),
5789            turn_id: turn_id.clone(),
5790            timestamp,
5791            event: kind,
5792        };
5793        if let Some(store) = &self.thread_store {
5794            store.append_item_event(thread_id, &item_event).await?;
5795        }
5796        self.remember_thread_item_event(&item_event).await?;
5797        Ok(item_event)
5798    }
5799
5800    async fn next_thread_item_event_seq(&self, thread_id: &ThreadId) -> anyhow::Result<u64> {
5801        self.ensure_thread_item_cache(thread_id).await?;
5802        Ok(self
5803            .thread_item_cache
5804            .lock()
5805            .await
5806            .next_item_event_seq(thread_id))
5807    }
5808
5809    pub async fn thread_item_exists(
5810        &self,
5811        thread_id: &ThreadId,
5812        turn_id: &TurnId,
5813        item_id: &str,
5814    ) -> anyhow::Result<bool> {
5815        self.ensure_thread_item_cache(thread_id).await?;
5816        Ok(self
5817            .thread_item_cache
5818            .lock()
5819            .await
5820            .thread_item_exists(thread_id, turn_id, item_id))
5821    }
5822
5823    pub async fn current_reasoning_item_id(
5824        &self,
5825        thread_id: &ThreadId,
5826        turn_id: &TurnId,
5827    ) -> anyhow::Result<Option<String>> {
5828        self.ensure_thread_item_cache(thread_id).await?;
5829        Ok(self
5830            .thread_item_cache
5831            .lock()
5832            .await
5833            .current_reasoning_item_id(thread_id, turn_id))
5834    }
5835
5836    async fn remember_thread_item_event(&self, item_event: &ThreadItemEvent) -> anyhow::Result<()> {
5837        self.ensure_thread_item_cache(&item_event.thread_id).await?;
5838        self.thread_item_cache
5839            .lock()
5840            .await
5841            .remember_item_event(item_event);
5842        Ok(())
5843    }
5844
5845    pub async fn latest_transcript_item_index(
5846        &self,
5847        thread_id: &ThreadId,
5848        turn_id: &TurnId,
5849    ) -> anyhow::Result<Option<usize>> {
5850        self.ensure_thread_item_cache(thread_id).await?;
5851        Ok(self
5852            .thread_item_cache
5853            .lock()
5854            .await
5855            .latest_transcript_item_index(thread_id, turn_id))
5856    }
5857
5858    async fn next_transcript_item_index(
5859        &self,
5860        thread_id: &ThreadId,
5861        turn_id: &TurnId,
5862    ) -> anyhow::Result<usize> {
5863        self.ensure_thread_item_cache(thread_id).await?;
5864        Ok(self
5865            .thread_item_cache
5866            .lock()
5867            .await
5868            .next_transcript_item_index(thread_id, turn_id))
5869    }
5870
5871    async fn remember_transcript_item_index(
5872        &self,
5873        thread_id: &ThreadId,
5874        turn_id: &TurnId,
5875        item_index: usize,
5876    ) -> anyhow::Result<()> {
5877        self.ensure_thread_item_cache(thread_id).await?;
5878        self.thread_item_cache
5879            .lock()
5880            .await
5881            .remember_transcript_item_index(thread_id, turn_id, item_index);
5882        Ok(())
5883    }
5884
5885    async fn ensure_thread_item_cache(&self, thread_id: &ThreadId) -> anyhow::Result<()> {
5886        if self
5887            .thread_item_cache
5888            .lock()
5889            .await
5890            .contains_thread(thread_id)
5891        {
5892            return Ok(());
5893        }
5894
5895        let snapshot = if let Some(store) = &self.thread_store {
5896            store.load_thread(thread_id).await?
5897        } else {
5898            None
5899        };
5900        self.thread_item_cache.lock().await.ensure_thread(
5901            thread_id,
5902            ThreadItemCacheEntry::from_snapshot(snapshot.as_ref()),
5903        );
5904        Ok(())
5905    }
5906
5907    pub(crate) async fn persist_turn_item(
5908        &self,
5909        thread_id: &ThreadId,
5910        turn_id: &TurnId,
5911        item: &TranscriptItem,
5912    ) -> anyhow::Result<()> {
5913        let item_index = self.next_transcript_item_index(thread_id, turn_id).await?;
5914        let timestamp = OffsetDateTime::now_utc();
5915        self.emit(RoderEvent::TranscriptItemAppended(TranscriptItemAppended {
5916            thread_id: thread_id.clone(),
5917            turn_id: turn_id.clone(),
5918            item_type: match item {
5919                TranscriptItem::UserMessage(_) => "user_message",
5920                TranscriptItem::AssistantMessage(_) => "assistant_message",
5921                TranscriptItem::ReasoningSummary(_) => "reasoning_summary",
5922                TranscriptItem::ToolCall(_) => "tool_call",
5923                TranscriptItem::ToolResult(_) => "tool_result",
5924                TranscriptItem::FileChange(_) => "file_change",
5925                TranscriptItem::ContextCompaction(_) => "context_compaction",
5926                TranscriptItem::Error(_) => "error",
5927                TranscriptItem::ProviderMetadata(_) => "provider_metadata",
5928            }
5929            .to_string(),
5930            item_index: Some(item_index),
5931            item: Some(item.clone()),
5932            timestamp,
5933        }))
5934        .await;
5935        self.remember_transcript_item_index(thread_id, turn_id, item_index)
5936            .await?;
5937        Ok(())
5938    }
5939}
5940
5941fn transcript_has_images(transcript: &[TranscriptItem]) -> bool {
5942    transcript.iter().any(|item| {
5943        matches!(
5944            item,
5945            TranscriptItem::UserMessage(message) if !message.images.is_empty()
5946        )
5947    })
5948}
5949
5950fn transcript_has_task_ledger(transcript: &[TranscriptItem]) -> bool {
5951    transcript.iter().any(|item| {
5952        matches!(
5953            item,
5954            TranscriptItem::ToolResult(result)
5955                if result.name.as_deref() == Some(TASK_LEDGER_TOOL_NAME) && !result.is_error
5956        )
5957    })
5958}
5959
5960fn task_ledger_completion_prompt(transcript: &[TranscriptItem]) -> Option<String> {
5961    let latest = transcript.iter().rev().find_map(|item| match item {
5962        TranscriptItem::ToolResult(result)
5963            if result.name.as_deref() == Some(TASK_LEDGER_TOOL_NAME) && !result.is_error =>
5964        {
5965            Some(result.result.as_str())
5966        }
5967        _ => None,
5968    })?;
5969    if !task_ledger_has_open_items(latest) {
5970        return None;
5971    }
5972
5973    let mut ledger = latest.chars().take(1500).collect::<String>();
5974    if latest.chars().nth(1500).is_some() {
5975        ledger.push_str("...");
5976    }
5977    Some(format!(
5978        "Task Ledger Completion Required: the latest task ledger still has pending or in-progress items. Do not provide a final answer yet. Use tools to complete the remaining scoreable work, create or update any required output files, then call `{TASK_LEDGER_TOOL_NAME}` with every task completed and evidence before finalizing.\n\nLatest ledger:\n{ledger}"
5979    ))
5980}
5981
5982fn task_ledger_deadline_completion_prompt(
5983    remaining_seconds: u64,
5984    reserve_seconds: u64,
5985    completion_prompt: &str,
5986) -> String {
5987    format!(
5988        "Eval deadline scoreable completion: {remaining_seconds} seconds remain in the {reserve_seconds}-second finalization reserve. Do not browse, search, or start slow work. Use the available tools now to create or update the required scoreable output files, run only a quick local check if needed, then update the task ledger to completed before finalizing.\n\n{completion_prompt}"
5989    )
5990}
5991
5992fn task_ledger_scoreable_checkpoint_prompt(
5993    remaining_seconds: u64,
5994    completion_prompt: &str,
5995) -> String {
5996    format!(
5997        "Scoreable Output Checkpoint: {remaining_seconds} seconds remain before the eval deadline. Before any further research, browsing, or long commands, use tools now to ensure the required output file(s) exist with the best evidence-backed answer, even if provisional. If a scoreable file already exists, read it and preserve that candidate unless you have stronger task-specific evidence for a replacement. Do not overwrite a plausible dated, historical, or local-evidence candidate with a current live-page, partial-coverage, or weaker guess merely to refresh the checkpoint. You may continue refining afterward, but do not apologize or finalize until the scoreable file exists and the task ledger is updated.\n\n{completion_prompt}"
5998    )
5999}
6000
6001fn task_ledger_has_open_items(ledger: &str) -> bool {
6002    ledger.lines().any(|line| {
6003        let line = line.trim_start();
6004        line.starts_with("- pending:") || line.starts_with("- in_progress:")
6005    })
6006}
6007
6008fn turn_deadline_for_config(cfg: &RuntimeConfig) -> Option<OffsetDateTime> {
6009    cfg.turn_deadline_seconds
6010        .filter(|seconds| *seconds > 0)
6011        .map(|seconds| OffsetDateTime::now_utc() + Duration::seconds(seconds as i64))
6012}
6013
6014fn deadline_expired(deadline: OffsetDateTime) -> bool {
6015    OffsetDateTime::now_utc() >= deadline
6016}
6017
6018pub(crate) fn deadline_remaining_seconds(deadline: Option<OffsetDateTime>) -> Option<u64> {
6019    let deadline = deadline?;
6020    if deadline <= OffsetDateTime::now_utc() {
6021        return Some(0);
6022    }
6023    Some(
6024        (deadline - OffsetDateTime::now_utc())
6025            .unsigned_abs()
6026            .as_secs()
6027            .max(1),
6028    )
6029}
6030
6031fn deadline_instant(deadline: OffsetDateTime) -> tokio::time::Instant {
6032    let now = OffsetDateTime::now_utc();
6033    if deadline <= now {
6034        return tokio::time::Instant::now();
6035    }
6036    tokio::time::Instant::now() + (deadline - now).unsigned_abs()
6037}
6038
6039fn inference_timeout_deadline(
6040    deadline: Option<OffsetDateTime>,
6041    runtime_profile: RuntimeProfile,
6042    task_ledger_required: bool,
6043    reserve_seconds: u64,
6044    finalization_requested: bool,
6045    task_ledger_scoreable_checkpoints: u8,
6046    transcript: &[TranscriptItem],
6047) -> Option<(OffsetDateTime, InferenceTimeoutAction)> {
6048    let deadline = deadline?;
6049    if runtime_profile == RuntimeProfile::Eval
6050        && task_ledger_required
6051        && !finalization_requested
6052        && task_ledger_scoreable_checkpoints < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
6053        && TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS > reserve_seconds
6054        && task_ledger_completion_prompt(transcript).is_some()
6055    {
6056        let checkpoint_deadline =
6057            deadline - Duration::seconds(TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS as i64);
6058        if checkpoint_deadline > OffsetDateTime::now_utc() {
6059            return Some((
6060                checkpoint_deadline,
6061                InferenceTimeoutAction::ScoreableCheckpoint,
6062            ));
6063        }
6064    }
6065    if runtime_profile == RuntimeProfile::Eval && !finalization_requested {
6066        return Some((
6067            deadline - Duration::seconds(reserve_seconds as i64),
6068            InferenceTimeoutAction::Finalization,
6069        ));
6070    }
6071    Some((deadline, InferenceTimeoutAction::Finalization))
6072}
6073
6074fn turn_partial_result(transcript: &[TranscriptItem]) -> String {
6075    let tool_results = transcript
6076        .iter()
6077        .filter(|item| matches!(item, TranscriptItem::ToolResult(_)))
6078        .count();
6079    let assistant_messages = transcript
6080        .iter()
6081        .filter(|item| matches!(item, TranscriptItem::AssistantMessage(_)))
6082        .count();
6083    format!(
6084        "partial turn state: {} transcript items, {assistant_messages} assistant messages, {tool_results} tool results",
6085        transcript.len()
6086    )
6087}
6088
6089fn reasoning_for_model(cfg: &RuntimeConfig, model: &str) -> ReasoningConfig {
6090    let level = effective_reasoning_for_model(cfg, model);
6091    match level.as_str() {
6092        "" | REASONING_NONE => ReasoningConfig::default(),
6093        level => ReasoningConfig {
6094            enabled: true,
6095            level: Some(level.to_string()),
6096        },
6097    }
6098}
6099
6100fn server_side_compaction_threshold(cfg: &RuntimeConfig, model: &str) -> Option<u32> {
6101    let entry = lookup_model(model)?;
6102    if !entry.supports_compaction {
6103        return None;
6104    }
6105    cfg.auto_compact_token_limit
6106        .or_else(|| {
6107            model_profile_for_model(cfg, model).and_then(|profile| profile.auto_compact_token_limit)
6108        })
6109        .or(Some(entry.auto_compact_token_limit))
6110        .filter(|threshold| *threshold > 0)
6111}
6112
6113pub(crate) fn tool_search_for_provider_model(
6114    cfg: &RuntimeConfig,
6115    provider: &str,
6116    model: &str,
6117) -> ToolSearchConfig {
6118    let mut resolved = cfg.tool_search.clone();
6119    if let Some(provider_config) = cfg.provider_tool_search.get(provider) {
6120        provider_config.apply_to(&mut resolved);
6121    }
6122    if let Some(model_config) = cfg.model_tool_search.get(model) {
6123        model_config.apply_to(&mut resolved);
6124    }
6125    resolved
6126}
6127
6128fn parallel_tool_calls_for_model(cfg: &RuntimeConfig, model: &str) -> bool {
6129    cfg.model_parallel_tool_calls
6130        .get(model)
6131        .copied()
6132        .or_else(|| {
6133            model_profile_for_model(cfg, model).and_then(|profile| profile.parallel_tool_calls)
6134        })
6135        .unwrap_or(true)
6136}
6137
6138/// Count the children an `agent_swarm` call will launch from its arguments
6139/// (item-based spawns plus resumes). Lenient: returns 0 on malformed input.
6140fn agent_swarm_child_count(arguments: &str) -> usize {
6141    serde_json::from_str::<roder_api::subagents::AgentSwarmRequest>(arguments)
6142        .map(|request| request.items.len() + request.resume_agent_ids.len())
6143        .unwrap_or(0)
6144}
6145
6146/// Parse `completed`/`failed`/`aborted` counts from an `<agent_swarm_result>`
6147/// summary line, e.g. `<summary>completed: 2, failed: 1</summary>`. Omitted
6148/// buckets default to 0. Returns `None` when the text is not a swarm result.
6149fn parse_swarm_counts(text: &str) -> Option<(usize, usize, usize)> {
6150    if !text.contains("<agent_swarm_result>") {
6151        return None;
6152    }
6153    let bucket = |label: &str| -> usize {
6154        let needle = format!("{label}: ");
6155        text.find(&needle)
6156            .map(|start| start + needle.len())
6157            .map(|start| {
6158                text[start..]
6159                    .chars()
6160                    .take_while(|c| c.is_ascii_digit())
6161                    .collect::<String>()
6162            })
6163            .and_then(|digits| digits.parse().ok())
6164            .unwrap_or(0)
6165    };
6166    Some((bucket("completed"), bucket("failed"), bucket("aborted")))
6167}
6168
6169fn effective_reasoning_for_model(cfg: &RuntimeConfig, model: &str) -> String {
6170    let base_reasoning = default_effective_reasoning_for_model(cfg, model);
6171    if cfg.dynamic_workflows.effort_profile == DynamicWorkflowEffortProfile::Ultracode {
6172        return ultracode_reasoning_level_for_model(
6173            model,
6174            &cfg.speed_policy.ultracode_reasoning,
6175            &base_reasoning,
6176        );
6177    }
6178    base_reasoning
6179}
6180
6181fn default_effective_reasoning_for_model(cfg: &RuntimeConfig, model: &str) -> String {
6182    let Some(entry) = lookup_model(model) else {
6183        return cfg
6184            .reasoning
6185            .clone()
6186            .unwrap_or_else(|| REASONING_NONE.to_string());
6187    };
6188    if entry.supported_reasoning.is_empty() {
6189        return REASONING_NONE.to_string();
6190    }
6191    cfg.reasoning
6192        .as_deref()
6193        .filter(|reasoning| {
6194            entry
6195                .supported_reasoning
6196                .iter()
6197                .any(|option| option.effort == *reasoning)
6198        })
6199        .map(str::to_string)
6200        .or_else(|| {
6201            model_profile_for_model(cfg, model)
6202                .and_then(|profile| profile.reasoning.orientation)
6203                .filter(|reasoning| {
6204                    entry
6205                        .supported_reasoning
6206                        .iter()
6207                        .any(|option| option.effort == reasoning)
6208                })
6209        })
6210        .unwrap_or_else(|| entry.default_reasoning.to_string())
6211}
6212
6213fn validate_reasoning_effort(model: &str, effort: &str) -> anyhow::Result<()> {
6214    if effort == REASONING_NONE && !model_supports_reasoning(model, effort) {
6215        return Ok(());
6216    }
6217    let Some(entry) = lookup_model(model) else {
6218        return Ok(());
6219    };
6220    if entry
6221        .supported_reasoning
6222        .iter()
6223        .any(|option| option.effort == effort)
6224    {
6225        Ok(())
6226    } else {
6227        anyhow::bail!("model {model} does not support reasoning effort {effort}")
6228    }
6229}
6230
6231fn validate_runtime_config_reasoning(cfg: &RuntimeConfig) -> anyhow::Result<()> {
6232    let Some(reasoning) = cfg.reasoning.as_deref() else {
6233        return Ok(());
6234    };
6235    let Some(entry) = lookup_model(&cfg.default_model) else {
6236        return Ok(());
6237    };
6238    if entry.provider != PROVIDER_GEMINI {
6239        return Ok(());
6240    }
6241    validate_reasoning_effort(&cfg.default_model, reasoning)
6242}
6243
6244fn validate_runtime_inference_router_config(
6245    registry: &ExtensionRegistry,
6246    cfg: &RuntimeConfig,
6247) -> anyhow::Result<()> {
6248    if !cfg.inference_router.enabled {
6249        return Ok(());
6250    }
6251    let Some(router_id) = cfg.inference_router.router_id.as_deref() else {
6252        anyhow::bail!("inference_router.enabled requires inference_router.router");
6253    };
6254    if registry.inference_router(router_id).is_some() {
6255        return Ok(());
6256    }
6257    let available = registry
6258        .inference_routers
6259        .iter()
6260        .map(|router| router.id())
6261        .collect::<Vec<_>>()
6262        .join(", ");
6263    if available.is_empty() {
6264        anyhow::bail!("inference router {router_id:?} is not registered");
6265    }
6266    anyhow::bail!(
6267        "inference router {router_id:?} is not registered; available routers: {available}"
6268    );
6269}
6270
6271fn model_supports_reasoning(model: &str, effort: &str) -> bool {
6272    lookup_model(model)
6273        .map(|entry| {
6274            entry
6275                .supported_reasoning
6276                .iter()
6277                .any(|option| option.effort == effort)
6278        })
6279        .unwrap_or(false)
6280}
6281
6282fn is_final_answer_phase(phase: Option<&str>) -> bool {
6283    phase.is_none_or(|phase| phase.is_empty() || phase == FINAL_ANSWER_PHASE)
6284}
6285
6286fn edit_tool_for_model<'a>(cfg: &'a RuntimeConfig, model: &'a str) -> Option<&'a str> {
6287    cfg.model_edit_tools
6288        .get(model)
6289        .map(String::as_str)
6290        .or_else(|| {
6291            cfg.model_profiles
6292                .get(model)
6293                .and_then(|profile| profile.edit_tool.as_deref())
6294        })
6295        .or_else(|| lookup_model(model).and_then(|entry| entry.edit_tool))
6296        .or(Some(EDIT_TOOL_EDIT))
6297}
6298
6299fn model_profile_for_model(cfg: &RuntimeConfig, model: &str) -> Option<ModelHarnessProfile> {
6300    cfg.model_profiles
6301        .get(model)
6302        .cloned()
6303        .or_else(|| built_in_model_profile(model))
6304}
6305
6306pub(crate) fn allowlist_permits(allowlist: &[String], tool_name: &str) -> bool {
6307    allowlist.is_empty() || allowlist.iter().any(|allowed| allowed == tool_name)
6308}
6309
6310/// Provider-aware profile resolution for the active turn.
6311///
6312/// Many model ids are shared across providers (for example Cursor proxies
6313/// `claude-opus-4-8`). Resolving the harness profile by id alone picks the
6314/// first catalog entry, which assigns cross-provider ids the wrong family and
6315/// instruction overlay (e.g. a `cursor/claude-opus-4-8` turn would otherwise
6316/// inherit the Anthropic overlay). Prefer the explicit `(provider, model)`
6317/// catalog entry, keeping user-configured profile overrides as the top
6318/// precedence.
6319fn model_profile_for_provider_model(
6320    cfg: &RuntimeConfig,
6321    provider: &str,
6322    model: &str,
6323) -> Option<ModelHarnessProfile> {
6324    cfg.model_profiles
6325        .get(model)
6326        .cloned()
6327        .or_else(|| built_in_model_profile_for_provider(provider, model))
6328}
6329
6330fn schema_policy_for_model(profile: Option<&ModelHarnessProfile>) -> ModelSchemaPolicy {
6331    profile
6332        .map(|profile| profile.schema_policy)
6333        .unwrap_or_default()
6334}
6335
6336fn model_profile_segment_metadata(
6337    profile: Option<&ModelHarnessProfile>,
6338    provider: &str,
6339    model: &str,
6340    segment: &str,
6341) -> serde_json::Value {
6342    serde_json::json!({
6343        "kind": MODEL_PROFILE_TRACE_KIND,
6344        "segment": segment,
6345        "provider": provider,
6346        "model": model,
6347        "profileModel": profile.map(|profile| profile.model.as_str()).unwrap_or(model),
6348        "providerFamily": profile.map(|profile| profile.provider_family),
6349        "editTool": profile.and_then(|profile| profile.edit_tool.as_deref()),
6350        "schemaPolicy": profile.map(|profile| profile.schema_policy),
6351        "instructionOverlay": profile.map(|profile| profile.instruction_overlay),
6352        "parallelToolCalls": profile.and_then(|profile| profile.parallel_tool_calls),
6353        "autoCompactTokenLimit": profile.and_then(|profile| profile.auto_compact_token_limit),
6354    })
6355}
6356
6357fn model_switch_summary(
6358    transcript: &[TranscriptItem],
6359    profile: Option<&ModelHarnessProfile>,
6360    provider: &str,
6361    model: &str,
6362    tools: &[roder_api::tools::ToolSpec],
6363) -> Option<String> {
6364    let previous = latest_model_profile_segment(transcript)?;
6365    let previous_model = previous
6366        .get("model")
6367        .and_then(serde_json::Value::as_str)
6368        .unwrap_or_default();
6369    let previous_provider = previous
6370        .get("provider")
6371        .and_then(serde_json::Value::as_str)
6372        .unwrap_or_default();
6373    if previous_model == model && previous_provider == provider {
6374        return None;
6375    }
6376
6377    let previous_profile = previous
6378        .get("profileModel")
6379        .and_then(serde_json::Value::as_str)
6380        .unwrap_or(previous_model);
6381    let current_profile = profile
6382        .map(|profile| profile.model.as_str())
6383        .unwrap_or(model);
6384    let previous_edit_tool = previous
6385        .get("editTool")
6386        .and_then(serde_json::Value::as_str)
6387        .unwrap_or("none");
6388    let current_edit_tool = profile
6389        .and_then(|profile| profile.edit_tool.as_deref())
6390        .unwrap_or("none");
6391    let tool_names = tools
6392        .iter()
6393        .map(|tool| tool.name.as_str())
6394        .take(12)
6395        .collect::<Vec<_>>()
6396        .join(", ");
6397    Some(format!(
6398        "{MODEL_SWITCH_SUMMARY_PREFIX} previous profile {previous_provider}/{previous_profile} used edit tool {previous_edit_tool}. Current profile {provider}/{current_profile} uses edit tool {current_edit_tool}. Available tools now: {}.",
6399        if tool_names.is_empty() {
6400            "none"
6401        } else {
6402            &tool_names
6403        }
6404    ))
6405}
6406
6407fn latest_model_profile_segment(transcript: &[TranscriptItem]) -> Option<&serde_json::Value> {
6408    transcript.iter().rev().find_map(|item| {
6409        let TranscriptItem::ProviderMetadata(value) = item else {
6410            return None;
6411        };
6412        (value.get("kind").and_then(serde_json::Value::as_str) == Some(MODEL_PROFILE_TRACE_KIND))
6413            .then_some(value)
6414    })
6415}
6416
6417pub fn validate_edit_tool(value: &str) -> anyhow::Result<()> {
6418    match value.trim() {
6419        EDIT_TOOL_PATCH | EDIT_TOOL_EDIT => Ok(()),
6420        _ => anyhow::bail!(
6421            "unsupported edit_tool {value:?}; allowed values: {EDIT_TOOL_PATCH}, {EDIT_TOOL_EDIT}"
6422        ),
6423    }
6424}
6425
6426fn should_persist_thread_event(thread_id: &str) -> bool {
6427    !is_synthetic_event_thread_id(thread_id)
6428}
6429
6430#[cfg(test)]
6431#[path = "runtime/item_identity_tests.rs"]
6432mod item_identity_tests;
6433
6434#[cfg(test)]
6435#[path = "runtime/codex_v2_tests.rs"]
6436mod codex_v2_tests;
6437
6438#[cfg(test)]
6439#[path = "runtime/codex_v2_lifecycle_tests.rs"]
6440mod codex_v2_lifecycle_tests;
6441
6442#[cfg(test)]
6443mod tests {
6444    use super::*;
6445    use futures::stream;
6446    use roder_api::catalog::{
6447        PROVIDER_MOCK, REASONING_HIGH, REASONING_LOW, REASONING_MEDIUM, REASONING_MINIMAL,
6448        REASONING_NONE, REASONING_XHIGH,
6449    };
6450    use roder_api::extension::ExtensionRegistryBuilder;
6451    use roder_api::inference::{
6452        CompletionMetadata, InferenceCapabilities, InferenceEngine, InferenceEventStream,
6453        InferenceProviderContext, InferenceTurnContext, MessageDelta, ModelDescriptor,
6454        ModelInstructionOverlay, ModelProfileReasoning, ModelSchemaPolicy, ProviderFamily,
6455        ReasoningEffortDescriptor,
6456    };
6457    use roder_api::inference_routing::{
6458        InferenceRouter, InferenceRoutingContext, InferenceRoutingDecision, InferenceRoutingOutcome,
6459    };
6460    use roder_api::thread::ThreadStoreFactory;
6461    use roder_api::tools::{ToolContributor, ToolExecutor, ToolSpec};
6462    use roder_ext_jsonl_thread_store::store::JsonlThreadStoreFactory;
6463    use std::sync::Mutex as StdMutex;
6464
6465    fn test_workspace() -> String {
6466        std::env::current_dir().unwrap().display().to_string()
6467    }
6468
6469    #[test]
6470    fn interactive_explicit_turn_deadline_has_bounded_remaining_time() {
6471        let config = RuntimeConfig {
6472            runtime_profile: RuntimeProfile::Interactive,
6473            turn_deadline_seconds: Some(60),
6474            ..RuntimeConfig::default()
6475        };
6476
6477        let deadline = turn_deadline_for_config(&config).expect("configured turn deadline");
6478        let remaining = deadline_remaining_seconds(Some(deadline));
6479
6480        assert!(
6481            matches!(remaining, Some(1..=60)),
6482            "interactive deadline must be positive and no longer than configured: {remaining:?}"
6483        );
6484    }
6485
6486    #[test]
6487    fn interactive_without_turn_deadline_stays_unbounded() {
6488        let config = RuntimeConfig {
6489            runtime_profile: RuntimeProfile::Interactive,
6490            turn_deadline_seconds: None,
6491            ..RuntimeConfig::default()
6492        };
6493
6494        assert_eq!(turn_deadline_for_config(&config), None);
6495    }
6496
6497    struct MetadataMissingStore;
6498
6499    #[async_trait::async_trait]
6500    impl ThreadStore for MetadataMissingStore {
6501        fn id(&self) -> roder_api::thread::ThreadStoreId {
6502            "metadata-missing-store".to_string()
6503        }
6504
6505        async fn create_thread(&self, metadata: ThreadMetadata) -> anyhow::Result<ThreadMetadata> {
6506            Ok(metadata)
6507        }
6508
6509        async fn list_threads(&self) -> anyhow::Result<Vec<ThreadMetadata>> {
6510            Ok(Vec::new())
6511        }
6512
6513        async fn load_thread(
6514            &self,
6515            _thread_id: &ThreadId,
6516        ) -> anyhow::Result<Option<ThreadSnapshot>> {
6517            Ok(Some(ThreadSnapshot {
6518                metadata: None,
6519                ..ThreadSnapshot::default()
6520            }))
6521        }
6522
6523        async fn append_event(
6524            &self,
6525            _thread_id: &ThreadId,
6526            _envelope: &EventEnvelope,
6527        ) -> anyhow::Result<()> {
6528            Ok(())
6529        }
6530    }
6531
6532    struct MetadataMissingStoreFactory;
6533
6534    impl ThreadStoreFactory for MetadataMissingStoreFactory {
6535        fn id(&self) -> roder_api::thread::ThreadStoreId {
6536            "metadata-missing-store".to_string()
6537        }
6538
6539        fn create(&self) -> Arc<dyn ThreadStore> {
6540            Arc::new(MetadataMissingStore)
6541        }
6542    }
6543
6544    #[test]
6545    fn synthetic_app_server_events_are_not_thread_events() {
6546        for thread_id in ["app-server", "runtime", "thread-workflow"] {
6547            assert!(!should_persist_thread_event(thread_id));
6548        }
6549        assert!(should_persist_thread_event("thread-discovery"));
6550        assert!(should_persist_thread_event("thread-plan"));
6551        assert!(should_persist_thread_event("thread-process"));
6552        assert!(should_persist_thread_event("thread-1"));
6553    }
6554
6555    #[test]
6556    fn server_side_compaction_uses_catalog_ninety_percent_default() {
6557        assert_eq!(
6558            server_side_compaction_threshold(&RuntimeConfig::default(), "gpt-5.5"),
6559            Some(945_000)
6560        );
6561        assert_eq!(
6562            server_side_compaction_threshold(&RuntimeConfig::default(), "gpt-5.3-codex-spark"),
6563            Some(115_200)
6564        );
6565    }
6566
6567    #[test]
6568    fn server_side_compaction_respects_explicit_config_override() {
6569        let cfg = RuntimeConfig {
6570            auto_compact_token_limit: Some(123_456),
6571            ..RuntimeConfig::default()
6572        };
6573
6574        assert_eq!(
6575            server_side_compaction_threshold(&cfg, "gpt-5.5"),
6576            Some(123_456)
6577        );
6578    }
6579
6580    #[tokio::test]
6581    async fn pre_request_compaction_runs_when_server_side_model_is_at_context_window() {
6582        let captured = Arc::new(StdMutex::new(None));
6583        let mut builder = ExtensionRegistryBuilder::new();
6584        builder.inference_engine(Arc::new(CapturingEngine {
6585            request: captured.clone(),
6586        }));
6587        let thread_root = std::env::temp_dir().join(format!(
6588            "roder-pre-request-compaction-{}",
6589            uuid::Uuid::new_v4()
6590        ));
6591        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
6592            base_path: thread_root.clone(),
6593        }));
6594        let runtime = Arc::new(
6595            Runtime::new(
6596                builder.build().unwrap(),
6597                RuntimeConfig {
6598                    default_provider: PROVIDER_MOCK.to_string(),
6599                    default_model: "gpt-5.5".to_string(),
6600                    file_backed_dynamic_context: true,
6601                    ..RuntimeConfig::default()
6602                },
6603            )
6604            .unwrap(),
6605        );
6606        let thread_id = runtime
6607            .create_thread(Some("Pre-request compaction".to_string()))
6608            .await
6609            .unwrap()
6610            .thread_id;
6611        let old_turn = "old-turn".to_string();
6612        runtime
6613            .persist_turn_item(
6614                &thread_id,
6615                &old_turn,
6616                &TranscriptItem::UserMessage(UserMessage::text("old context ".repeat(4_300_000))),
6617            )
6618            .await
6619            .unwrap();
6620
6621        let mut events = runtime.subscribe_events();
6622        runtime
6623            .start_turn(StartTurnRequest {
6624                thread_id: thread_id.clone(),
6625                message: "continue".to_string(),
6626                images: Vec::new(),
6627                provider_override: None,
6628                model_override: None,
6629                reasoning_override: None,
6630                workspace: test_workspace(),
6631                instructions: InstructionBundle::default(),
6632                developer_context: None,
6633                task_ledger_required: false,
6634                service_tier_override: None,
6635            })
6636            .await
6637            .unwrap();
6638        loop {
6639            let envelope = tokio::time::timeout(std::time::Duration::from_secs(5), events.recv())
6640                .await
6641                .unwrap()
6642                .unwrap();
6643            if envelope.thread_id.as_deref() == Some(&thread_id)
6644                && matches!(envelope.event, RoderEvent::TurnCompleted(_))
6645            {
6646                break;
6647            }
6648        }
6649
6650        let request = captured.lock().unwrap().clone().unwrap();
6651        assert!(
6652            matches!(
6653                request.transcript.first(),
6654                Some(TranscriptItem::ContextCompaction(_))
6655            ),
6656            "provider request should start with a local emergency compaction item"
6657        );
6658        assert!(
6659            request.transcript.len() < 4,
6660            "provider request should not replay the full oversized prior transcript: {:?}",
6661            request.transcript
6662        );
6663
6664        let _ = std::fs::remove_dir_all(thread_root);
6665    }
6666
6667    #[tokio::test]
6668    async fn continue_after_context_window_failure_compacts_before_provider_request() {
6669        let captured = Arc::new(StdMutex::new(None));
6670        let mut builder = ExtensionRegistryBuilder::new();
6671        builder.inference_engine(Arc::new(CapturingEngine {
6672            request: captured.clone(),
6673        }));
6674        let thread_root = std::env::temp_dir().join(format!(
6675            "roder-context-failure-continue-{}",
6676            uuid::Uuid::new_v4()
6677        ));
6678        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
6679            base_path: thread_root.clone(),
6680        }));
6681        let runtime = Arc::new(
6682            Runtime::new(
6683                builder.build().unwrap(),
6684                RuntimeConfig {
6685                    default_provider: PROVIDER_MOCK.to_string(),
6686                    default_model: "gpt-5.5".to_string(),
6687                    file_backed_dynamic_context: true,
6688                    ..RuntimeConfig::default()
6689                },
6690            )
6691            .unwrap(),
6692        );
6693        let thread_id = runtime
6694            .create_thread(Some("Context failure continue".to_string()))
6695            .await
6696            .unwrap()
6697            .thread_id;
6698        let failed_turn = "failed-turn".to_string();
6699        runtime
6700            .persist_turn_item(
6701                &thread_id,
6702                &failed_turn,
6703                &TranscriptItem::UserMessage(UserMessage::text("old work ".repeat(10_000))),
6704            )
6705            .await
6706            .unwrap();
6707        runtime
6708            .persist_turn_item(
6709                &thread_id,
6710                &failed_turn,
6711                &TranscriptItem::Error(ErrorRecord {
6712                    message: "Your input exceeds the context window of this model. Please adjust your input and try again."
6713                        .to_string(),
6714                }),
6715            )
6716            .await
6717            .unwrap();
6718
6719        let mut events = runtime.subscribe_events();
6720        runtime
6721            .start_turn(StartTurnRequest {
6722                thread_id: thread_id.clone(),
6723                message: "continue".to_string(),
6724                images: Vec::new(),
6725                provider_override: None,
6726                model_override: None,
6727                reasoning_override: None,
6728                workspace: test_workspace(),
6729                instructions: InstructionBundle::default(),
6730                developer_context: None,
6731                task_ledger_required: false,
6732                service_tier_override: None,
6733            })
6734            .await
6735            .unwrap();
6736        loop {
6737            let envelope = tokio::time::timeout(std::time::Duration::from_secs(5), events.recv())
6738                .await
6739                .unwrap()
6740                .unwrap();
6741            if envelope.thread_id.as_deref() == Some(&thread_id)
6742                && matches!(envelope.event, RoderEvent::TurnCompleted(_))
6743            {
6744                break;
6745            }
6746        }
6747
6748        let request = captured.lock().unwrap().clone().unwrap();
6749        assert!(
6750            matches!(
6751                request.transcript.first(),
6752                Some(TranscriptItem::ContextCompaction(_))
6753            ),
6754            "provider request after context-window failure should start with local compaction"
6755        );
6756        assert!(
6757            request
6758                .transcript
6759                .iter()
6760                .any(|item| matches!(item, TranscriptItem::UserMessage(message) if message.text == "continue")),
6761            "current continue prompt must be preserved: {:?}",
6762            request.transcript
6763        );
6764        assert!(
6765            !request.transcript.iter().any(
6766                |item| matches!(item, TranscriptItem::Error(error) if error.message.contains("context window"))
6767            ),
6768            "raw prior context-window error should be summarized, not replayed: {:?}",
6769            request.transcript
6770        );
6771
6772        let _ = std::fs::remove_dir_all(thread_root);
6773    }
6774
6775    #[tokio::test]
6776    async fn workspace_for_thread_falls_back_when_metadata_is_missing() {
6777        let workspace = test_workspace();
6778        let mut builder = ExtensionRegistryBuilder::new();
6779        builder.inference_engine(Arc::new(FakeInferenceEngine));
6780        builder.thread_store_factory(Arc::new(MetadataMissingStoreFactory));
6781        let runtime = Runtime::new(
6782            builder.build().unwrap(),
6783            RuntimeConfig {
6784                workspace: Some(workspace.clone()),
6785                ..RuntimeConfig::default()
6786            },
6787        )
6788        .unwrap();
6789
6790        let resolved = runtime
6791            .workspace_for_thread(&ThreadId::from("thread-workflow"))
6792            .await
6793            .unwrap();
6794
6795        assert_eq!(resolved, workspace);
6796    }
6797
6798    #[tokio::test]
6799    async fn automations_can_create_project_thread_with_model_overrides() {
6800        let runtime = Runtime::fake().unwrap();
6801        let workspace = std::env::temp_dir().join("project");
6802        let metadata = runtime
6803            .create_thread_with(CreateThreadRequest {
6804                title: Some("Automation: nightly status".to_string()),
6805                workspace: workspace.display().to_string(),
6806                workspace_id: None,
6807                root_id: None,
6808                provider: Some("mock".to_string()),
6809                model: Some("mock".to_string()),
6810                selection_mode: None,
6811                tool_allowlist: Vec::new(),
6812                developer_instructions: None,
6813                external_tools: Vec::new(),
6814                runner: None,
6815            })
6816            .await
6817            .unwrap();
6818
6819        assert_eq!(
6820            metadata.title.as_deref(),
6821            Some("Automation: nightly status")
6822        );
6823        assert_eq!(metadata.workspace, workspace.display().to_string());
6824        assert_eq!(metadata.provider.as_deref(), Some("mock"));
6825        assert_eq!(metadata.model.as_deref(), Some("mock"));
6826    }
6827
6828    #[tokio::test]
6829    async fn prior_provider_compaction_boundary_is_used_on_next_turn() {
6830        let captured = Arc::new(StdMutex::new(None));
6831        let mut builder = ExtensionRegistryBuilder::new();
6832        builder.inference_engine(Arc::new(CapturingEngine {
6833            request: captured.clone(),
6834        }));
6835        let thread_root = std::env::temp_dir().join(format!(
6836            "roder-provider-compaction-boundary-{}",
6837            uuid::Uuid::new_v4()
6838        ));
6839        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
6840            base_path: thread_root.clone(),
6841        }));
6842        let runtime = Arc::new(
6843            Runtime::new(
6844                builder.build().unwrap(),
6845                RuntimeConfig {
6846                    default_provider: PROVIDER_MOCK.to_string(),
6847                    default_model: "gpt-5.5".to_string(),
6848                    ..RuntimeConfig::default()
6849                },
6850            )
6851            .unwrap(),
6852        );
6853        let thread_id = runtime
6854            .create_thread(Some("Provider compaction boundary".to_string()))
6855            .await
6856            .unwrap()
6857            .thread_id;
6858        let old_turn = "old-turn".to_string();
6859        runtime
6860            .persist_turn_item(
6861                &thread_id,
6862                &old_turn,
6863                &TranscriptItem::UserMessage(UserMessage::text(
6864                    "old history that must not be replayed after provider compaction",
6865                )),
6866            )
6867            .await
6868            .unwrap();
6869        runtime
6870            .persist_turn_item(
6871                &thread_id,
6872                &old_turn,
6873                &TranscriptItem::AssistantMessage(AssistantMessage {
6874                    text: "old answer".to_string(),
6875                    phase: None,
6876                }),
6877            )
6878            .await
6879            .unwrap();
6880        runtime
6881            .persist_turn_item(
6882                &thread_id,
6883                &old_turn,
6884                &TranscriptItem::ProviderMetadata(serde_json::json!({
6885                    "output": [{
6886                        "id": "cmp_1",
6887                        "type": "compaction",
6888                        "encrypted_content": "opaque-state"
6889                    }]
6890                })),
6891            )
6892            .await
6893            .unwrap();
6894        runtime
6895            .persist_turn_item(
6896                &thread_id,
6897                &old_turn,
6898                &TranscriptItem::AssistantMessage(AssistantMessage {
6899                    text: "after compact".to_string(),
6900                    phase: None,
6901                }),
6902            )
6903            .await
6904            .unwrap();
6905
6906        let mut events = runtime.subscribe_events();
6907        runtime
6908            .start_turn(StartTurnRequest {
6909                thread_id: thread_id.clone(),
6910                message: "continue".to_string(),
6911                images: Vec::new(),
6912                provider_override: None,
6913                model_override: None,
6914                reasoning_override: None,
6915                workspace: test_workspace(),
6916                instructions: InstructionBundle::default(),
6917                developer_context: None,
6918                task_ledger_required: false,
6919                service_tier_override: None,
6920            })
6921            .await
6922            .unwrap();
6923        loop {
6924            let envelope = tokio::time::timeout(std::time::Duration::from_secs(5), events.recv())
6925                .await
6926                .unwrap()
6927                .unwrap();
6928            if envelope.thread_id.as_deref() == Some(&thread_id)
6929                && matches!(envelope.event, RoderEvent::TurnCompleted(_))
6930            {
6931                break;
6932            }
6933        }
6934
6935        let request = captured.lock().unwrap().clone().unwrap();
6936        let compaction_idx = request.transcript.iter().position(|item| {
6937            matches!(
6938                item,
6939                TranscriptItem::ProviderMetadata(metadata)
6940                    if crate::compaction::provider_metadata_has_compaction(metadata)
6941            )
6942        });
6943        assert!(
6944            compaction_idx.is_some(),
6945            "next provider request should include the provider compaction item: {:?}",
6946            request.transcript
6947        );
6948        // Skill/context injectors may prepend non-history items, but no prior-turn
6949        // conversation may appear before the latest provider compaction boundary.
6950        let history_before_boundary =
6951            request
6952                .transcript
6953                .iter()
6954                .take(compaction_idx.unwrap())
6955                .any(|item| match item {
6956                    TranscriptItem::AssistantMessage(_)
6957                    | TranscriptItem::ToolCall(_)
6958                    | TranscriptItem::ToolResult(_) => true,
6959                    TranscriptItem::UserMessage(message) => {
6960                        !message.text.contains("<skills>") && message.text != "continue"
6961                    }
6962                    _ => false,
6963                });
6964        assert!(
6965            !history_before_boundary,
6966            "pre-provider-compaction conversation must not be replayed: {:?}",
6967            request.transcript
6968        );
6969        assert!(
6970            !request.transcript.iter().any(|item| {
6971                matches!(
6972                    item,
6973                    TranscriptItem::UserMessage(message)
6974                        if message.text.contains("old history that must not be replayed")
6975                )
6976            }),
6977            "pre-provider-compaction history must not be replayed: {:?}",
6978            request.transcript
6979        );
6980        assert!(
6981            request.transcript.iter().any(|item| {
6982                matches!(
6983                    item,
6984                    TranscriptItem::UserMessage(message) if message.text == "continue"
6985                )
6986            }),
6987            "current continue prompt must be preserved: {:?}",
6988            request.transcript
6989        );
6990        assert_eq!(
6991            request.runtime.auto_compact_token_limit,
6992            Some(945_000),
6993            "gpt-5.5 should keep server-side auto compaction configured"
6994        );
6995
6996        let _ = std::fs::remove_dir_all(thread_root);
6997    }
6998
6999    #[test]
7000    fn server_side_compaction_is_only_enabled_for_supported_models() {
7001        let cfg = RuntimeConfig {
7002            auto_compact_token_limit: Some(123_456),
7003            ..RuntimeConfig::default()
7004        };
7005
7006        assert_eq!(server_side_compaction_threshold(&cfg, "mock"), None);
7007        assert_eq!(
7008            server_side_compaction_threshold(&cfg, "codex-auto-review"),
7009            None
7010        );
7011    }
7012
7013    #[test]
7014    fn reasoning_is_disabled_for_models_without_reasoning_support() {
7015        let cfg = RuntimeConfig {
7016            reasoning: Some(REASONING_HIGH.to_string()),
7017            ..RuntimeConfig::default()
7018        };
7019
7020        assert_eq!(
7021            effective_reasoning_for_model(&cfg, "claude-haiku-4-5-20251001"),
7022            REASONING_NONE
7023        );
7024        assert_eq!(
7025            reasoning_for_model(&cfg, "claude-haiku-4-5-20251001"),
7026            ReasoningConfig::default()
7027        );
7028    }
7029
7030    #[test]
7031    fn unsupported_configured_reasoning_falls_back_to_model_default() {
7032        let cfg = RuntimeConfig {
7033            reasoning: Some(REASONING_MINIMAL.to_string()),
7034            ..RuntimeConfig::default()
7035        };
7036
7037        assert_eq!(
7038            effective_reasoning_for_model(&cfg, "gpt-5.5"),
7039            REASONING_MEDIUM
7040        );
7041    }
7042
7043    #[test]
7044    fn unsupported_configured_gemini_reasoning_is_rejected() {
7045        let mut builder = ExtensionRegistryBuilder::new();
7046        builder.inference_engine(std::sync::Arc::new(FakeInferenceEngine));
7047
7048        let err = match Runtime::new(
7049            builder.build().unwrap(),
7050            RuntimeConfig {
7051                default_model: "gemini-3.5-flash".to_string(),
7052                reasoning: Some(REASONING_XHIGH.to_string()),
7053                ..RuntimeConfig::default()
7054            },
7055        ) {
7056            Ok(_) => panic!("expected unsupported Gemini reasoning to be rejected"),
7057            Err(err) => err,
7058        };
7059
7060        assert!(
7061            err.to_string()
7062                .contains("model gemini-3.5-flash does not support reasoning effort xhigh")
7063        );
7064    }
7065
7066    #[tokio::test]
7067    async fn selecting_none_for_non_reasoning_model_preserves_stored_preference() {
7068        let runtime = Runtime::new(
7069            Runtime::fake().unwrap().registry,
7070            RuntimeConfig {
7071                reasoning: Some(REASONING_HIGH.to_string()),
7072                ..RuntimeConfig::default()
7073            },
7074        )
7075        .unwrap();
7076
7077        let cfg = runtime
7078            .select_provider(
7079                roder_api::catalog::PROVIDER_MOCK.to_string(),
7080                Some("claude-haiku-4-5-20251001".to_string()),
7081                Some(REASONING_NONE.to_string()),
7082            )
7083            .await
7084            .unwrap();
7085
7086        assert_eq!(cfg.reasoning.as_deref(), Some(REASONING_HIGH));
7087        assert_eq!(runtime.effective_reasoning().await, REASONING_NONE);
7088    }
7089
7090    #[tokio::test]
7091    async fn selecting_none_for_model_that_supports_none_updates_preference() {
7092        let runtime = Runtime::new(
7093            Runtime::fake().unwrap().registry,
7094            RuntimeConfig {
7095                reasoning: Some(REASONING_HIGH.to_string()),
7096                ..RuntimeConfig::default()
7097            },
7098        )
7099        .unwrap();
7100
7101        let cfg = runtime
7102            .select_provider(
7103                roder_api::catalog::PROVIDER_MOCK.to_string(),
7104                Some("mock".to_string()),
7105                Some(REASONING_NONE.to_string()),
7106            )
7107            .await
7108            .unwrap();
7109
7110        assert_eq!(cfg.reasoning.as_deref(), Some(REASONING_NONE));
7111    }
7112
7113    #[test]
7114    fn parallel_tool_calls_default_on_with_model_override() {
7115        assert!(parallel_tool_calls_for_model(
7116            &RuntimeConfig::default(),
7117            "custom-model"
7118        ));
7119
7120        let cfg = RuntimeConfig {
7121            model_parallel_tool_calls: std::collections::HashMap::from([(
7122                "custom-model".to_string(),
7123                false,
7124            )]),
7125            ..RuntimeConfig::default()
7126        };
7127
7128        assert!(!parallel_tool_calls_for_model(&cfg, "custom-model"));
7129        assert!(parallel_tool_calls_for_model(&cfg, "other-model"));
7130    }
7131
7132    #[test]
7133    fn profile_parallel_tool_calls_applies_between_config_and_default() {
7134        let cfg = RuntimeConfig {
7135            model_profiles: std::collections::HashMap::from([(
7136                "gpt-5.5".to_string(),
7137                test_model_profile("gpt-5.5"),
7138            )]),
7139            ..RuntimeConfig::default()
7140        };
7141
7142        assert!(!parallel_tool_calls_for_model(&cfg, "gpt-5.5"));
7143
7144        let cfg = RuntimeConfig {
7145            model_parallel_tool_calls: std::collections::HashMap::from([(
7146                "gpt-5.5".to_string(),
7147                true,
7148            )]),
7149            ..cfg
7150        };
7151
7152        assert!(parallel_tool_calls_for_model(&cfg, "gpt-5.5"));
7153    }
7154
7155    struct CapturingEngine {
7156        request: Arc<StdMutex<Option<AgentInferenceRequest>>>,
7157    }
7158
7159    #[async_trait::async_trait]
7160    impl InferenceEngine for CapturingEngine {
7161        fn id(&self) -> String {
7162            roder_api::catalog::PROVIDER_MOCK.to_string()
7163        }
7164
7165        fn capabilities(&self) -> InferenceCapabilities {
7166            InferenceCapabilities::coding_agent_default()
7167        }
7168
7169        async fn list_models(
7170            &self,
7171            _ctx: InferenceProviderContext<'_>,
7172        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7173            Ok(roder_api::catalog::models_for_provider(
7174                roder_api::catalog::PROVIDER_MOCK,
7175                true,
7176            ))
7177        }
7178
7179        async fn stream_turn(
7180            &self,
7181            _ctx: InferenceTurnContext<'_>,
7182            request: AgentInferenceRequest,
7183        ) -> anyhow::Result<InferenceEventStream> {
7184            *self.request.lock().unwrap() = Some(request);
7185            Ok(Box::pin(stream::iter(vec![
7186                Ok(InferenceEvent::MessageDelta(MessageDelta {
7187                    text: "done".to_string(),
7188                    phase: None,
7189                })),
7190                Ok(InferenceEvent::Completed(CompletionMetadata {
7191                    stop_reason: Some("stop".to_string()),
7192                    provider_response_id: None,
7193                })),
7194            ])))
7195        }
7196    }
7197
7198    struct RoutingCaptureEngine {
7199        id: &'static str,
7200        models: Vec<ModelDescriptor>,
7201        requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
7202    }
7203
7204    #[async_trait::async_trait]
7205    impl InferenceEngine for RoutingCaptureEngine {
7206        fn id(&self) -> String {
7207            self.id.to_string()
7208        }
7209
7210        fn capabilities(&self) -> InferenceCapabilities {
7211            InferenceCapabilities::coding_agent_default()
7212        }
7213
7214        async fn list_models(
7215            &self,
7216            _ctx: InferenceProviderContext<'_>,
7217        ) -> anyhow::Result<Vec<ModelDescriptor>> {
7218            Ok(self.models.clone())
7219        }
7220
7221        async fn stream_turn(
7222            &self,
7223            _ctx: InferenceTurnContext<'_>,
7224            request: AgentInferenceRequest,
7225        ) -> anyhow::Result<InferenceEventStream> {
7226            self.requests.lock().unwrap().push(request);
7227            Ok(Box::pin(stream::iter(vec![
7228                Ok(InferenceEvent::MessageDelta(MessageDelta {
7229                    text: "routed".to_string(),
7230                    phase: None,
7231                })),
7232                Ok(InferenceEvent::Completed(CompletionMetadata {
7233                    stop_reason: Some("stop".to_string()),
7234                    provider_response_id: None,
7235                })),
7236            ])))
7237        }
7238    }
7239
7240    struct StaticRouter {
7241        id: &'static str,
7242        decision: InferenceRoutingDecision,
7243        contexts: Arc<StdMutex<Vec<InferenceRoutingContext>>>,
7244    }
7245
7246    #[async_trait::async_trait]
7247    impl InferenceRouter for StaticRouter {
7248        fn id(&self) -> String {
7249            self.id.to_string()
7250        }
7251
7252        async fn route(
7253            &self,
7254            context: InferenceRoutingContext,
7255        ) -> anyhow::Result<InferenceRoutingDecision> {
7256            self.contexts.lock().unwrap().push(context);
7257            Ok(self.decision.clone())
7258        }
7259    }
7260
7261    fn routing_test_model(id: &str, supported_reasoning: &[&str]) -> ModelDescriptor {
7262        ModelDescriptor {
7263            id: id.to_string(),
7264            name: id.to_string(),
7265            context_window: Some(128_000),
7266            default_reasoning: supported_reasoning
7267                .first()
7268                .map(|effort| (*effort).to_string()),
7269            supported_reasoning: supported_reasoning
7270                .iter()
7271                .map(|effort| ReasoningEffortDescriptor {
7272                    effort: (*effort).to_string(),
7273                    description: format!("{effort} reasoning"),
7274                })
7275                .collect(),
7276        }
7277    }
7278
7279    struct TaskLedgerCompletionGateEngine {
7280        calls: StdMutex<u32>,
7281        requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
7282    }
7283
7284    #[async_trait::async_trait]
7285    impl InferenceEngine for TaskLedgerCompletionGateEngine {
7286        fn id(&self) -> String {
7287            roder_api::catalog::PROVIDER_MOCK.to_string()
7288        }
7289
7290        fn capabilities(&self) -> InferenceCapabilities {
7291            InferenceCapabilities::coding_agent_default()
7292        }
7293
7294        async fn list_models(
7295            &self,
7296            _ctx: InferenceProviderContext<'_>,
7297        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7298            Ok(roder_api::catalog::models_for_provider(
7299                roder_api::catalog::PROVIDER_MOCK,
7300                true,
7301            ))
7302        }
7303
7304        async fn stream_turn(
7305            &self,
7306            _ctx: InferenceTurnContext<'_>,
7307            request: AgentInferenceRequest,
7308        ) -> anyhow::Result<InferenceEventStream> {
7309            self.requests.lock().unwrap().push(request);
7310            let mut calls = self.calls.lock().unwrap();
7311            *calls += 1;
7312            let events = match *calls {
7313                1 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7314                    id: "ledger-open".to_string(),
7315                    name: TASK_LEDGER_TOOL_NAME.to_string(),
7316                    arguments: serde_json::json!({
7317                        "tasks": [
7318                            {
7319                                "id": "inspect",
7320                                "content": "Inspect local assets",
7321                                "status": "completed",
7322                                "evidence": "listed workspace"
7323                            },
7324                            {
7325                                "id": "write",
7326                                "content": "Write /app/result.txt",
7327                                "status": "pending"
7328                            }
7329                        ],
7330                        "requireCompletionEvidence": true
7331                    })
7332                    .to_string(),
7333                }))],
7334                3 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7335                    id: "ledger-complete".to_string(),
7336                    name: TASK_LEDGER_TOOL_NAME.to_string(),
7337                    arguments: serde_json::json!({
7338                        "tasks": [
7339                            {
7340                                "id": "inspect",
7341                                "content": "Inspect local assets",
7342                                "status": "completed",
7343                                "evidence": "listed workspace"
7344                            },
7345                            {
7346                                "id": "write",
7347                                "content": "Write /app/result.txt",
7348                                "status": "completed",
7349                                "evidence": "wrote answer"
7350                            }
7351                        ],
7352                        "requireCompletionEvidence": true
7353                    })
7354                    .to_string(),
7355                }))],
7356                _ => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
7357                    text: "final".to_string(),
7358                    phase: None,
7359                }))],
7360            };
7361            Ok(Box::pin(stream::iter(events.into_iter().chain(
7362                std::iter::once(Ok(InferenceEvent::Completed(CompletionMetadata {
7363                    stop_reason: Some("stop".to_string()),
7364                    provider_response_id: None,
7365                }))),
7366            ))))
7367        }
7368    }
7369
7370    struct VerificationGateEngine {
7371        calls: StdMutex<u32>,
7372    }
7373
7374    #[async_trait::async_trait]
7375    impl InferenceEngine for VerificationGateEngine {
7376        fn id(&self) -> String {
7377            roder_api::catalog::PROVIDER_MOCK.to_string()
7378        }
7379
7380        fn capabilities(&self) -> InferenceCapabilities {
7381            InferenceCapabilities::coding_agent_default()
7382        }
7383
7384        async fn list_models(
7385            &self,
7386            _ctx: InferenceProviderContext<'_>,
7387        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7388            Ok(roder_api::catalog::models_for_provider(
7389                roder_api::catalog::PROVIDER_MOCK,
7390                true,
7391            ))
7392        }
7393
7394        async fn stream_turn(
7395            &self,
7396            _ctx: InferenceTurnContext<'_>,
7397            request: AgentInferenceRequest,
7398        ) -> anyhow::Result<InferenceEventStream> {
7399            let mut calls = self.calls.lock().unwrap();
7400            *calls += 1;
7401            let events = match *calls {
7402                1 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7403                    id: "write-1".to_string(),
7404                    name: "write_file".to_string(),
7405                    arguments: serde_json::json!({
7406                        "path": "src/lib.rs",
7407                        "content": "pub fn answer() -> u8 { 42 }\n"
7408                    })
7409                    .to_string(),
7410                }))],
7411                2 => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
7412                    text: "done too early".to_string(),
7413                    phase: None,
7414                }))],
7415                3 if request.transcript.iter().any(|item| {
7416                    matches!(
7417                        item,
7418                        TranscriptItem::UserMessage(message)
7419                            if message.text.contains("Verification gate blocked final completion")
7420                    )
7421                }) =>
7422                {
7423                    vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7424                        id: "verify-1".to_string(),
7425                        name: crate::verification_gate::VERIFICATION_TOOL_NAME.to_string(),
7426                        arguments: serde_json::json!({
7427                            "originalTask": "write code",
7428                            "changedFiles": ["src/lib.rs"],
7429                            "toolEvidence": ["write_file wrote src/lib.rs"],
7430                            "testsRun": ["cargo test -p roder-core verification_gate"],
7431                            "openGaps": [],
7432                            "status": "completed"
7433                        })
7434                        .to_string(),
7435                    }))]
7436                }
7437                _ => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
7438                    text: "verified final".to_string(),
7439                    phase: None,
7440                }))],
7441            };
7442            Ok(Box::pin(stream::iter(events.into_iter().chain(
7443                std::iter::once(Ok(InferenceEvent::Completed(CompletionMetadata {
7444                    stop_reason: Some("stop".to_string()),
7445                    provider_response_id: None,
7446                }))),
7447            ))))
7448        }
7449    }
7450
7451    struct SpeedPolicyEngine {
7452        calls: StdMutex<u32>,
7453        requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
7454    }
7455
7456    #[async_trait::async_trait]
7457    impl InferenceEngine for SpeedPolicyEngine {
7458        fn id(&self) -> String {
7459            roder_api::catalog::PROVIDER_MOCK.to_string()
7460        }
7461
7462        fn capabilities(&self) -> InferenceCapabilities {
7463            InferenceCapabilities::coding_agent_default()
7464        }
7465
7466        async fn list_models(
7467            &self,
7468            _ctx: InferenceProviderContext<'_>,
7469        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7470            Ok(roder_api::catalog::models_for_provider(
7471                roder_api::catalog::PROVIDER_MOCK,
7472                true,
7473            ))
7474        }
7475
7476        async fn stream_turn(
7477            &self,
7478            _ctx: InferenceTurnContext<'_>,
7479            request: AgentInferenceRequest,
7480        ) -> anyhow::Result<InferenceEventStream> {
7481            self.requests.lock().unwrap().push(request.clone());
7482            let mut calls = self.calls.lock().unwrap();
7483            *calls += 1;
7484            let events = match *calls {
7485                1 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7486                    id: "write-1".to_string(),
7487                    name: "write_file".to_string(),
7488                    arguments: serde_json::json!({
7489                        "path": "src/lib.rs",
7490                        "content": "pub fn answer() -> u8 { 42 }\n"
7491                    })
7492                    .to_string(),
7493                }))],
7494                3 if request.transcript.iter().any(|item| {
7495                    matches!(
7496                        item,
7497                        TranscriptItem::UserMessage(message)
7498                            if message.text.contains("Verification gate blocked final completion")
7499                    )
7500                }) =>
7501                {
7502                    vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7503                        id: "verify-1".to_string(),
7504                        name: crate::verification_gate::VERIFICATION_TOOL_NAME.to_string(),
7505                        arguments: serde_json::json!({
7506                            "originalTask": "write code",
7507                            "changedFiles": ["src/lib.rs"],
7508                            "toolEvidence": ["write_file wrote src/lib.rs"],
7509                            "testsRun": ["cargo test -p roder-core speed_policy"],
7510                            "openGaps": [],
7511                            "status": "completed"
7512                        })
7513                        .to_string(),
7514                    }))]
7515                }
7516                _ => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
7517                    text: "done".to_string(),
7518                    phase: None,
7519                }))],
7520            };
7521            Ok(Box::pin(stream::iter(events.into_iter().chain(
7522                std::iter::once(Ok(InferenceEvent::Completed(CompletionMetadata {
7523                    stop_reason: Some("stop".to_string()),
7524                    provider_response_id: None,
7525                }))),
7526            ))))
7527        }
7528    }
7529
7530    struct SwitchCaptureEngine {
7531        requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
7532    }
7533
7534    #[async_trait::async_trait]
7535    impl InferenceEngine for SwitchCaptureEngine {
7536        fn id(&self) -> String {
7537            roder_api::catalog::PROVIDER_MOCK.to_string()
7538        }
7539
7540        fn capabilities(&self) -> InferenceCapabilities {
7541            InferenceCapabilities::coding_agent_default()
7542        }
7543
7544        async fn list_models(
7545            &self,
7546            _ctx: InferenceProviderContext<'_>,
7547        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7548            Ok(roder_api::catalog::models_for_provider(
7549                roder_api::catalog::PROVIDER_MOCK,
7550                true,
7551            ))
7552        }
7553
7554        async fn stream_turn(
7555            &self,
7556            _ctx: InferenceTurnContext<'_>,
7557            request: AgentInferenceRequest,
7558        ) -> anyhow::Result<InferenceEventStream> {
7559            self.requests.lock().unwrap().push(request);
7560            Ok(Box::pin(stream::iter(vec![
7561                Ok(InferenceEvent::MessageDelta(MessageDelta {
7562                    text: "done".to_string(),
7563                    phase: None,
7564                })),
7565                Ok(InferenceEvent::Completed(CompletionMetadata {
7566                    stop_reason: Some("stop".to_string()),
7567                    provider_response_id: None,
7568                })),
7569            ])))
7570        }
7571    }
7572
7573    struct DeadlineEngine;
7574
7575    #[async_trait::async_trait]
7576    impl InferenceEngine for DeadlineEngine {
7577        fn id(&self) -> String {
7578            roder_api::catalog::PROVIDER_MOCK.to_string()
7579        }
7580
7581        fn capabilities(&self) -> InferenceCapabilities {
7582            InferenceCapabilities::coding_agent_default()
7583        }
7584
7585        async fn list_models(
7586            &self,
7587            _ctx: InferenceProviderContext<'_>,
7588        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7589            Ok(Vec::new())
7590        }
7591
7592        async fn stream_turn(
7593            &self,
7594            _ctx: InferenceTurnContext<'_>,
7595            _request: AgentInferenceRequest,
7596        ) -> anyhow::Result<InferenceEventStream> {
7597            Ok(Box::pin(stream::once(async {
7598                tokio::time::sleep(std::time::Duration::from_secs(60)).await;
7599                Ok(InferenceEvent::MessageDelta(MessageDelta {
7600                    text: "too late".to_string(),
7601                    phase: None,
7602                }))
7603            })))
7604        }
7605    }
7606
7607    struct WriteFileContributor;
7608
7609    impl ToolContributor for WriteFileContributor {
7610        fn id(&self) -> String {
7611            "test-write".to_string()
7612        }
7613
7614        fn contribute(&self, registry: &mut ToolRegistry) -> anyhow::Result<()> {
7615            registry.register(Arc::new(WriteFileTool))
7616        }
7617    }
7618
7619    struct WriteFileTool;
7620
7621    #[async_trait::async_trait]
7622    impl ToolExecutor for WriteFileTool {
7623        fn spec(&self) -> ToolSpec {
7624            ToolSpec {
7625                name: "write_file".to_string(),
7626                description: "Write a test file.".to_string(),
7627                parameters: serde_json::json!({
7628                    "type": "object",
7629                    "properties": {
7630                        "path": { "type": "string" },
7631                        "content": { "type": "string" }
7632                    },
7633                    "required": ["path", "content"],
7634                    "additionalProperties": false
7635                }),
7636            }
7637        }
7638
7639        async fn execute(
7640            &self,
7641            _ctx: ToolExecutionContext,
7642            call: ToolCall,
7643        ) -> anyhow::Result<ToolResult> {
7644            let path = call
7645                .arguments
7646                .get("path")
7647                .and_then(serde_json::Value::as_str)
7648                .unwrap_or("src/lib.rs");
7649            Ok(ToolResult {
7650                id: call.id,
7651                name: call.name,
7652                text: format!("wrote {path}"),
7653                data: serde_json::json!({ "path": path }),
7654                is_error: false,
7655            })
7656        }
7657    }
7658
7659    struct ProfileToolContributor;
7660
7661    impl ToolContributor for ProfileToolContributor {
7662        fn id(&self) -> String {
7663            "profile-tools".to_string()
7664        }
7665
7666        fn contribute(&self, registry: &mut ToolRegistry) -> anyhow::Result<()> {
7667            for name in ["apply_patch", "edit", "multi_edit", "write_file"] {
7668                registry.register(Arc::new(ProfileTool {
7669                    name: name.to_string(),
7670                }))?;
7671            }
7672            Ok(())
7673        }
7674    }
7675
7676    struct ProfileTool {
7677        name: String,
7678    }
7679
7680    #[async_trait::async_trait]
7681    impl ToolExecutor for ProfileTool {
7682        fn spec(&self) -> ToolSpec {
7683            ToolSpec {
7684                name: self.name.clone(),
7685                description: format!("{} test tool", self.name),
7686                parameters: serde_json::json!({
7687                    "type": "object",
7688                    "properties": {
7689                        "path": { "type": "string" },
7690                        "content": { "type": "string" }
7691                    },
7692                    "required": ["path", "content"],
7693                    "additionalProperties": false
7694                }),
7695            }
7696        }
7697
7698        async fn execute(
7699            &self,
7700            _ctx: ToolExecutionContext,
7701            call: ToolCall,
7702        ) -> anyhow::Result<ToolResult> {
7703            Ok(ToolResult {
7704                id: call.id,
7705                name: call.name,
7706                text: "ok".to_string(),
7707                data: serde_json::json!({}),
7708                is_error: false,
7709            })
7710        }
7711    }
7712
7713    fn test_model_profile(model: &str) -> ModelHarnessProfile {
7714        ModelHarnessProfile {
7715            model: model.to_string(),
7716            provider: roder_api::catalog::PROVIDER_OPENAI.to_string(),
7717            provider_family: ProviderFamily::OpenAi,
7718            edit_tool: Some(EDIT_TOOL_EDIT.to_string()),
7719            schema_policy: ModelSchemaPolicy::StandardRequiredFirst,
7720            instruction_overlay: ModelInstructionOverlay::IntuitiveContext,
7721            reasoning: ModelProfileReasoning {
7722                orientation: Some(REASONING_LOW.to_string()),
7723                execution: Some(REASONING_LOW.to_string()),
7724                verification: Some(REASONING_LOW.to_string()),
7725                recovery: Some(REASONING_LOW.to_string()),
7726            },
7727            parallel_tool_calls: Some(false),
7728            auto_compact_token_limit: Some(123_000),
7729        }
7730    }
7731
7732    async fn captured_profile_request(cfg: RuntimeConfig) -> AgentInferenceRequest {
7733        let captured = Arc::new(StdMutex::new(None));
7734        let mut builder = ExtensionRegistryBuilder::new();
7735        builder.inference_engine(Arc::new(CapturingEngine {
7736            request: captured.clone(),
7737        }));
7738        builder.tool_contributor(Arc::new(ProfileToolContributor));
7739        let runtime = Arc::new(Runtime::new(builder.build().unwrap(), cfg).unwrap());
7740        let mut rx = runtime.subscribe_events();
7741        let turn_id = runtime
7742            .start_turn(StartTurnRequest {
7743                thread_id: "thread-model-profile".to_string(),
7744                message: "use profile knobs".to_string(),
7745                images: Vec::new(),
7746                provider_override: None,
7747                model_override: None,
7748                reasoning_override: None,
7749                workspace: test_workspace(),
7750                instructions: InstructionBundle {
7751                    system: None,
7752                    developer: Some("base developer".to_string()),
7753                    developer_context: None,
7754                },
7755                developer_context: None,
7756                task_ledger_required: false,
7757                service_tier_override: None,
7758            })
7759            .await
7760            .unwrap();
7761
7762        tokio::time::timeout(std::time::Duration::from_secs(5), async {
7763            loop {
7764                let envelope = rx.recv().await.unwrap();
7765                if envelope.turn_id.as_deref() != Some(&turn_id) {
7766                    continue;
7767                }
7768                match envelope.event {
7769                    RoderEvent::TurnCompleted(_) => break,
7770                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7771                    _ => {}
7772                }
7773            }
7774        })
7775        .await
7776        .unwrap();
7777
7778        captured.lock().unwrap().clone().unwrap()
7779    }
7780
7781    struct ToolThenStopEngine {
7782        calls: StdMutex<u32>,
7783    }
7784
7785    #[async_trait::async_trait]
7786    impl InferenceEngine for ToolThenStopEngine {
7787        fn id(&self) -> String {
7788            roder_api::catalog::PROVIDER_MOCK.to_string()
7789        }
7790
7791        fn capabilities(&self) -> InferenceCapabilities {
7792            InferenceCapabilities::coding_agent_default()
7793        }
7794
7795        async fn list_models(
7796            &self,
7797            _ctx: InferenceProviderContext<'_>,
7798        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7799            Ok(roder_api::catalog::models_for_provider(
7800                roder_api::catalog::PROVIDER_MOCK,
7801                true,
7802            ))
7803        }
7804
7805        async fn stream_turn(
7806            &self,
7807            _ctx: InferenceTurnContext<'_>,
7808            _request: AgentInferenceRequest,
7809        ) -> anyhow::Result<InferenceEventStream> {
7810            let mut calls = self.calls.lock().unwrap();
7811            *calls += 1;
7812            let events = match *calls {
7813                1 => vec![
7814                    Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7815                        id: "write-1".to_string(),
7816                        name: "write_file".to_string(),
7817                        arguments: serde_json::json!({
7818                            "path": "src/lib.rs",
7819                            "content": "pub fn answer() -> u8 { 42 }\n"
7820                        })
7821                        .to_string(),
7822                    })),
7823                    Ok(InferenceEvent::Completed(CompletionMetadata {
7824                        stop_reason: Some("tool_use".to_string()),
7825                        provider_response_id: None,
7826                    })),
7827                ],
7828                _ => vec![
7829                    Ok(InferenceEvent::MessageDelta(MessageDelta {
7830                        text: "final".to_string(),
7831                        phase: None,
7832                    })),
7833                    Ok(InferenceEvent::Completed(CompletionMetadata {
7834                        stop_reason: Some("end_turn".to_string()),
7835                        provider_response_id: None,
7836                    })),
7837                ],
7838            };
7839            Ok(Box::pin(stream::iter(events)))
7840        }
7841    }
7842
7843    #[tokio::test]
7844    async fn turn_completed_reports_terminal_step_finish_reason() {
7845        let mut builder = ExtensionRegistryBuilder::new();
7846        builder.inference_engine(Arc::new(ToolThenStopEngine {
7847            calls: StdMutex::new(0),
7848        }));
7849        builder.tool_contributor(Arc::new(WriteFileContributor));
7850        let runtime = Arc::new(
7851            Runtime::new(
7852                builder.build().unwrap(),
7853                RuntimeConfig {
7854                    policy_mode: PolicyMode::Bypass,
7855                    agent_swarm_mode: false,
7856                    ultra_mode: false,
7857                    ..RuntimeConfig::default()
7858                },
7859            )
7860            .unwrap(),
7861        );
7862        let mut rx = runtime.subscribe_events();
7863        let turn_id = runtime
7864            .start_turn(StartTurnRequest {
7865                thread_id: "thread-finish-reason".to_string(),
7866                message: "write then finish".to_string(),
7867                images: Vec::new(),
7868                provider_override: None,
7869                model_override: None,
7870                reasoning_override: None,
7871                workspace: test_workspace(),
7872                instructions: InstructionBundle {
7873                    system: None,
7874                    developer: None,
7875                    developer_context: None,
7876                },
7877                developer_context: None,
7878                task_ledger_required: false,
7879                service_tier_override: None,
7880            })
7881            .await
7882            .unwrap();
7883
7884        let completed = tokio::time::timeout(std::time::Duration::from_secs(5), async {
7885            loop {
7886                let envelope = rx.recv().await.unwrap();
7887                if envelope.turn_id.as_deref() != Some(&turn_id) {
7888                    continue;
7889                }
7890                match envelope.event {
7891                    RoderEvent::TurnCompleted(event) => break event,
7892                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7893                    _ => {}
7894                }
7895            }
7896        })
7897        .await
7898        .unwrap();
7899
7900        // The mid-turn tool_use stop reason must not leak; the terminal
7901        // end_turn step decides the turn's finish reason.
7902        assert_eq!(completed.finish_reason.as_deref(), Some("stop"));
7903    }
7904
7905    #[tokio::test]
7906    async fn inference_router_selection_changes_request_model_and_records_event() {
7907        let default_requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7908        let routed_requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7909        let contexts = Arc::new(StdMutex::new(Vec::<InferenceRoutingContext>::new()));
7910        let selected = ModelSelection {
7911            provider: "routed-provider".to_string(),
7912            model: "routed-model".to_string(),
7913        };
7914        let default = ModelSelection {
7915            provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7916            model: "mock".to_string(),
7917        };
7918        let decision = InferenceRoutingDecision {
7919            reasoning: Some(ReasoningConfig {
7920                enabled: true,
7921                level: Some(REASONING_LOW.to_string()),
7922            }),
7923            confidence: Some(0.91),
7924            baseline: Some(default.clone()),
7925            matched_signals: vec![roder_api::inference_routing::InferenceRoutingSignal::new(
7926                "intent", "routine",
7927            )],
7928            ..InferenceRoutingDecision::selected("test-router", selected.clone(), "routine request")
7929        };
7930
7931        let mut builder = ExtensionRegistryBuilder::new();
7932        builder.inference_engine(Arc::new(RoutingCaptureEngine {
7933            id: roder_api::catalog::PROVIDER_MOCK,
7934            models: vec![routing_test_model("mock", &[REASONING_LOW])],
7935            requests: default_requests.clone(),
7936        }));
7937        builder.inference_engine(Arc::new(RoutingCaptureEngine {
7938            id: "routed-provider",
7939            models: vec![routing_test_model(
7940                "routed-model",
7941                &[REASONING_LOW, REASONING_MEDIUM],
7942            )],
7943            requests: routed_requests.clone(),
7944        }));
7945        builder.inference_router(Arc::new(StaticRouter {
7946            id: "test-router",
7947            decision,
7948            contexts: contexts.clone(),
7949        }));
7950        let thread_root =
7951            std::env::temp_dir().join(format!("roder-routing-auto-{}", uuid::Uuid::new_v4()));
7952        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
7953            base_path: thread_root.clone(),
7954        }));
7955        let runtime = Arc::new(
7956            Runtime::new(
7957                builder.build().unwrap(),
7958                RuntimeConfig {
7959                    default_provider: default.provider.clone(),
7960                    default_model: default.model.clone(),
7961                    ..RuntimeConfig::default()
7962                },
7963            )
7964            .unwrap(),
7965        );
7966        let thread_id = runtime
7967            .create_thread_with(CreateThreadRequest {
7968                title: Some("Routing auto".to_string()),
7969                workspace: test_workspace(),
7970                workspace_id: None,
7971                root_id: None,
7972                provider: Some(default.provider.clone()),
7973                model: Some(default.model.clone()),
7974                tool_allowlist: Vec::new(),
7975                developer_instructions: None,
7976                external_tools: Vec::new(),
7977                selection_mode: Some(ModelSelectionMode::auto(
7978                    "test-router:coding",
7979                    "test-router",
7980                    "Auto: Coding",
7981                    default.clone(),
7982                    Some("coding".to_string()),
7983                    None,
7984                )),
7985                runner: None,
7986            })
7987            .await
7988            .unwrap()
7989            .thread_id;
7990        let mut rx = runtime.subscribe_events();
7991        let turn_id = runtime
7992            .start_turn(StartTurnRequest {
7993                thread_id: thread_id.clone(),
7994                message: "small cleanup".to_string(),
7995                images: Vec::new(),
7996                provider_override: None,
7997                model_override: None,
7998                reasoning_override: None,
7999                workspace: test_workspace(),
8000                instructions: InstructionBundle::default(),
8001                developer_context: None,
8002                task_ledger_required: false,
8003                service_tier_override: None,
8004            })
8005            .await
8006            .unwrap();
8007
8008        let mut routing_event = None;
8009        let mut inference_started = None;
8010        tokio::time::timeout(std::time::Duration::from_secs(5), async {
8011            loop {
8012                let envelope = rx.recv().await.unwrap();
8013                if envelope.turn_id.as_deref() != Some(&turn_id) {
8014                    continue;
8015                }
8016                match envelope.event {
8017                    RoderEvent::InferenceRoutingDecision(event) => {
8018                        routing_event = Some(event);
8019                    }
8020                    RoderEvent::InferenceStarted(event) => {
8021                        inference_started = Some(event);
8022                    }
8023                    RoderEvent::TurnCompleted(_) => break,
8024                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8025                    _ => {}
8026                }
8027            }
8028        })
8029        .await
8030        .unwrap();
8031
8032        assert!(default_requests.lock().unwrap().is_empty());
8033        let routed_requests = routed_requests.lock().unwrap();
8034        assert_eq!(routed_requests.len(), 1);
8035        assert_eq!(routed_requests[0].model, selected);
8036        assert_eq!(
8037            routed_requests[0].reasoning.level.as_deref(),
8038            Some(REASONING_LOW)
8039        );
8040        assert_eq!(
8041            routed_requests[0].metadata["inferenceRouting"]["outcome"],
8042            "selected"
8043        );
8044
8045        let routing_event = routing_event.expect("routing decision event");
8046        assert_eq!(routing_event.default_selection, default);
8047        assert_eq!(routing_event.selected_selection, selected);
8048        assert_eq!(
8049            routing_event.decision.outcome,
8050            InferenceRoutingOutcome::Selected
8051        );
8052        assert_eq!(
8053            inference_started.expect("inference started event").model,
8054            selected
8055        );
8056
8057        let contexts = contexts.lock().unwrap();
8058        assert_eq!(contexts.len(), 1);
8059        assert_eq!(contexts[0].default_selection, default);
8060        assert_eq!(contexts[0].candidates.len(), 2);
8061        assert!(
8062            contexts[0]
8063                .signals
8064                .iter()
8065                .any(|signal| signal.key == "profile" && signal.value == "coding")
8066        );
8067        let _ = std::fs::remove_dir_all(thread_root);
8068    }
8069
8070    #[tokio::test]
8071    async fn inference_router_is_bypassed_for_explicit_selection() {
8072        let requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
8073        let contexts = Arc::new(StdMutex::new(Vec::<InferenceRoutingContext>::new()));
8074        let selected = ModelSelection {
8075            provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8076            model: "mock".to_string(),
8077        };
8078
8079        let mut builder = ExtensionRegistryBuilder::new();
8080        builder.inference_engine(Arc::new(RoutingCaptureEngine {
8081            id: roder_api::catalog::PROVIDER_MOCK,
8082            models: vec![routing_test_model("mock", &[REASONING_LOW])],
8083            requests: requests.clone(),
8084        }));
8085        builder.inference_router(Arc::new(StaticRouter {
8086            id: "test-router",
8087            decision: InferenceRoutingDecision::selected(
8088                "test-router",
8089                ModelSelection {
8090                    provider: "missing".to_string(),
8091                    model: "missing".to_string(),
8092                },
8093                "would route if called",
8094            ),
8095            contexts: contexts.clone(),
8096        }));
8097        let thread_root =
8098            std::env::temp_dir().join(format!("roder-routing-explicit-{}", uuid::Uuid::new_v4()));
8099        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8100            base_path: thread_root.clone(),
8101        }));
8102        let runtime = Arc::new(
8103            Runtime::new(
8104                builder.build().unwrap(),
8105                RuntimeConfig {
8106                    default_provider: selected.provider.clone(),
8107                    default_model: selected.model.clone(),
8108                    inference_router: RuntimeInferenceRouterConfig {
8109                        enabled: true,
8110                        router_id: Some("test-router".to_string()),
8111                    },
8112                    ..RuntimeConfig::default()
8113                },
8114            )
8115            .unwrap(),
8116        );
8117        let thread_id = runtime
8118            .create_thread_with(CreateThreadRequest {
8119                title: Some("Routing explicit".to_string()),
8120                workspace: test_workspace(),
8121                workspace_id: None,
8122                root_id: None,
8123                provider: Some(selected.provider.clone()),
8124                model: Some(selected.model.clone()),
8125                tool_allowlist: Vec::new(),
8126                developer_instructions: None,
8127                external_tools: Vec::new(),
8128                selection_mode: Some(ModelSelectionMode::auto(
8129                    "test-router:default",
8130                    "test-router",
8131                    "Auto",
8132                    selected.clone(),
8133                    None,
8134                    None,
8135                )),
8136                runner: None,
8137            })
8138            .await
8139            .unwrap()
8140            .thread_id;
8141        let mut rx = runtime.subscribe_events();
8142        let turn_id = runtime
8143            .start_turn(StartTurnRequest {
8144                thread_id,
8145                message: "use explicit selection".to_string(),
8146                images: Vec::new(),
8147                provider_override: Some(selected.provider.clone()),
8148                model_override: Some(selected.model.clone()),
8149                reasoning_override: None,
8150                workspace: test_workspace(),
8151                instructions: InstructionBundle::default(),
8152                developer_context: None,
8153                task_ledger_required: false,
8154                service_tier_override: None,
8155            })
8156            .await
8157            .unwrap();
8158
8159        let mut saw_routing_event = false;
8160        tokio::time::timeout(std::time::Duration::from_secs(5), async {
8161            loop {
8162                let envelope = rx.recv().await.unwrap();
8163                if envelope.turn_id.as_deref() != Some(&turn_id) {
8164                    continue;
8165                }
8166                match envelope.event {
8167                    RoderEvent::InferenceRoutingDecision(_) => {
8168                        saw_routing_event = true;
8169                    }
8170                    RoderEvent::TurnCompleted(_) => break,
8171                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8172                    _ => {}
8173                }
8174            }
8175        })
8176        .await
8177        .unwrap();
8178
8179        assert!(!saw_routing_event);
8180        assert!(contexts.lock().unwrap().is_empty());
8181        let requests = requests.lock().unwrap();
8182        assert_eq!(requests.len(), 1);
8183        assert_eq!(requests[0].model, selected);
8184        let _ = std::fs::remove_dir_all(thread_root);
8185    }
8186
8187    #[tokio::test]
8188    async fn inference_router_is_bypassed_for_manual_selection_mode() {
8189        let requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
8190        let contexts = Arc::new(StdMutex::new(Vec::<InferenceRoutingContext>::new()));
8191        let selected = ModelSelection {
8192            provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8193            model: "mock".to_string(),
8194        };
8195
8196        let mut builder = ExtensionRegistryBuilder::new();
8197        builder.inference_engine(Arc::new(RoutingCaptureEngine {
8198            id: roder_api::catalog::PROVIDER_MOCK,
8199            models: vec![routing_test_model("mock", &[REASONING_LOW])],
8200            requests: requests.clone(),
8201        }));
8202        builder.inference_router(Arc::new(StaticRouter {
8203            id: "test-router",
8204            decision: InferenceRoutingDecision::selected(
8205                "test-router",
8206                ModelSelection {
8207                    provider: "missing".to_string(),
8208                    model: "missing".to_string(),
8209                },
8210                "would route if called",
8211            ),
8212            contexts: contexts.clone(),
8213        }));
8214        let thread_root =
8215            std::env::temp_dir().join(format!("roder-routing-manual-{}", uuid::Uuid::new_v4()));
8216        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8217            base_path: thread_root.clone(),
8218        }));
8219        let runtime = Arc::new(
8220            Runtime::new(
8221                builder.build().unwrap(),
8222                RuntimeConfig {
8223                    default_provider: selected.provider.clone(),
8224                    default_model: selected.model.clone(),
8225                    inference_router: RuntimeInferenceRouterConfig {
8226                        enabled: true,
8227                        router_id: Some("test-router".to_string()),
8228                    },
8229                    ..RuntimeConfig::default()
8230                },
8231            )
8232            .unwrap(),
8233        );
8234        let thread_id = runtime
8235            .create_thread_with(CreateThreadRequest {
8236                title: Some("Routing manual".to_string()),
8237                workspace: test_workspace(),
8238                workspace_id: None,
8239                root_id: None,
8240                provider: Some(selected.provider.clone()),
8241                model: Some(selected.model.clone()),
8242                tool_allowlist: Vec::new(),
8243                developer_instructions: None,
8244                external_tools: Vec::new(),
8245                selection_mode: Some(ModelSelectionMode::manual(
8246                    selected.provider.clone(),
8247                    selected.model.clone(),
8248                    None,
8249                )),
8250                runner: None,
8251            })
8252            .await
8253            .unwrap()
8254            .thread_id;
8255        let mut rx = runtime.subscribe_events();
8256        let turn_id = runtime
8257            .start_turn(StartTurnRequest {
8258                thread_id,
8259                message: "use selected manual model".to_string(),
8260                images: Vec::new(),
8261                provider_override: None,
8262                model_override: None,
8263                reasoning_override: None,
8264                workspace: test_workspace(),
8265                instructions: InstructionBundle::default(),
8266                developer_context: None,
8267                task_ledger_required: false,
8268                service_tier_override: None,
8269            })
8270            .await
8271            .unwrap();
8272
8273        let mut saw_routing_event = false;
8274        tokio::time::timeout(std::time::Duration::from_secs(5), async {
8275            loop {
8276                let envelope = rx.recv().await.unwrap();
8277                if envelope.turn_id.as_deref() != Some(&turn_id) {
8278                    continue;
8279                }
8280                match envelope.event {
8281                    RoderEvent::InferenceRoutingDecision(_) => {
8282                        saw_routing_event = true;
8283                    }
8284                    RoderEvent::TurnCompleted(_) => break,
8285                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8286                    _ => {}
8287                }
8288            }
8289        })
8290        .await
8291        .unwrap();
8292
8293        assert!(!saw_routing_event);
8294        assert!(contexts.lock().unwrap().is_empty());
8295        let requests = requests.lock().unwrap();
8296        assert_eq!(requests.len(), 1);
8297        assert_eq!(requests[0].model, selected);
8298        let _ = std::fs::remove_dir_all(thread_root);
8299    }
8300
8301    #[test]
8302    fn enabled_inference_router_requires_registered_router() {
8303        let mut builder = ExtensionRegistryBuilder::new();
8304        builder.inference_engine(Arc::new(FakeInferenceEngine));
8305
8306        let err = match Runtime::new(
8307            builder.build().unwrap(),
8308            RuntimeConfig {
8309                inference_router: RuntimeInferenceRouterConfig {
8310                    enabled: true,
8311                    router_id: Some("missing-router".to_string()),
8312                },
8313                ..RuntimeConfig::default()
8314            },
8315        ) {
8316            Ok(_) => panic!("runtime should reject unknown inference router"),
8317            Err(err) => err,
8318        };
8319
8320        assert!(
8321            err.to_string()
8322                .contains("inference router \"missing-router\" is not registered")
8323        );
8324    }
8325
8326    #[tokio::test]
8327    async fn model_profile_routes_request_knobs_to_next_inference() {
8328        let request = captured_profile_request(RuntimeConfig {
8329            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8330            default_model: "gpt-5.5".to_string(),
8331            model_profiles: std::collections::HashMap::from([(
8332                "gpt-5.5".to_string(),
8333                test_model_profile("gpt-5.5"),
8334            )]),
8335            ..RuntimeConfig::default()
8336        })
8337        .await;
8338
8339        let tool_names = request
8340            .tools
8341            .iter()
8342            .map(|tool| tool.name.as_str())
8343            .collect::<Vec<_>>();
8344        assert!(tool_names.contains(&"apply_patch"));
8345        assert!(tool_names.contains(&"edit"));
8346        assert!(tool_names.contains(&"multi_edit"));
8347        assert!(tool_names.contains(&"write_file"));
8348        assert_eq!(request.reasoning.level.as_deref(), Some(REASONING_LOW));
8349        assert_eq!(request.runtime.parallel_tool_calls, Some(false));
8350        assert_eq!(request.runtime.auto_compact_token_limit, Some(123_000));
8351        assert!(
8352            request
8353                .instructions
8354                .developer
8355                .as_deref()
8356                .unwrap_or_default()
8357                .contains("Use the provided context as the current working set")
8358        );
8359        assert_eq!(
8360            request
8361                .metadata
8362                .pointer("/modelProfile/schemaPolicy")
8363                .and_then(serde_json::Value::as_str),
8364            Some("standard_required_first")
8365        );
8366    }
8367
8368    #[tokio::test]
8369    async fn ultra_reasoning_reaches_transport_state_and_enables_proactive_delegation() {
8370        let request = captured_profile_request(RuntimeConfig {
8371            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8372            default_model: "gpt-5.6-sol".to_string(),
8373            reasoning: Some(REASONING_ULTRA.to_string()),
8374            ..RuntimeConfig::default()
8375        })
8376        .await;
8377
8378        assert_eq!(request.reasoning.level.as_deref(), Some(REASONING_ULTRA));
8379        let developer = request
8380            .instructions
8381            .developer
8382            .as_deref()
8383            .expect("ultra developer instructions");
8384        assert!(developer.contains("Proactive multi-agent delegation is active"));
8385    }
8386
8387    #[tokio::test]
8388    async fn lower_sol_effort_requires_explicit_multi_agent_request() {
8389        let request = captured_profile_request(RuntimeConfig {
8390            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8391            default_model: "gpt-5.6-sol".to_string(),
8392            reasoning: Some(roder_api::catalog::REASONING_MEDIUM.to_string()),
8393            ..RuntimeConfig::default()
8394        })
8395        .await;
8396
8397        let developer = request
8398            .instructions
8399            .developer
8400            .as_deref()
8401            .expect("sol developer instructions");
8402        assert!(developer.contains("Do not spawn sub-agents unless"));
8403        assert!(!developer.contains("Proactive multi-agent delegation is active"));
8404    }
8405
8406    #[tokio::test]
8407    async fn luna_does_not_receive_codex_v2_multi_agent_policy() {
8408        let request = captured_profile_request(RuntimeConfig {
8409            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8410            default_model: "gpt-5.6-luna".to_string(),
8411            reasoning: Some(roder_api::catalog::REASONING_MAX.to_string()),
8412            ..RuntimeConfig::default()
8413        })
8414        .await;
8415
8416        let developer = request
8417            .instructions
8418            .developer
8419            .as_deref()
8420            .unwrap_or_default();
8421        assert!(!developer.contains("Do not spawn sub-agents unless"));
8422        assert!(!developer.contains("Proactive multi-agent delegation is active"));
8423    }
8424
8425    #[tokio::test]
8426    async fn ultra_mode_enables_proactive_multi_agent_for_any_model() {
8427        // Luna has no Ultra effort, but ultra mode still injects proactive policy.
8428        let request = captured_profile_request(RuntimeConfig {
8429            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8430            default_model: "gpt-5.6-luna".to_string(),
8431            reasoning: Some(roder_api::catalog::REASONING_MAX.to_string()),
8432            ultra_mode: true,
8433            ..RuntimeConfig::default()
8434        })
8435        .await;
8436
8437        let developer = request
8438            .instructions
8439            .developer
8440            .as_deref()
8441            .expect("ultra mode developer instructions");
8442        assert!(developer.contains("Proactive multi-agent delegation is active"));
8443        assert!(developer.contains("spawn_agent"));
8444    }
8445
8446    #[tokio::test]
8447    async fn ultra_mode_overrides_explicit_request_only_on_sol() {
8448        // Medium Sol effort is normally explicit-request-only; ultra mode flips
8449        // it to proactive without requiring Ultra reasoning effort.
8450        let request = captured_profile_request(RuntimeConfig {
8451            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8452            default_model: "gpt-5.6-sol".to_string(),
8453            reasoning: Some(roder_api::catalog::REASONING_MEDIUM.to_string()),
8454            ultra_mode: true,
8455            ..RuntimeConfig::default()
8456        })
8457        .await;
8458
8459        let developer = request
8460            .instructions
8461            .developer
8462            .as_deref()
8463            .expect("ultra mode sol developer instructions");
8464        assert!(developer.contains("Proactive multi-agent delegation is active"));
8465        assert!(!developer.contains("Do not spawn sub-agents unless"));
8466    }
8467
8468    #[tokio::test]
8469    async fn turn_developer_context_reaches_inference_and_does_not_persist() {
8470        let captured = Arc::new(StdMutex::new(None));
8471        let mut builder = ExtensionRegistryBuilder::new();
8472        builder.inference_engine(Arc::new(CapturingEngine {
8473            request: captured.clone(),
8474        }));
8475        let runtime = Arc::new(
8476            Runtime::new(
8477                builder.build().unwrap(),
8478                RuntimeConfig {
8479                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8480                    default_model: "gpt-5.5".to_string(),
8481                    ..RuntimeConfig::default()
8482                },
8483            )
8484            .unwrap(),
8485        );
8486
8487        async fn run_turn(runtime: &Arc<Runtime>, developer_context: Option<String>) {
8488            let mut rx = runtime.subscribe_events();
8489            let turn_id = runtime
8490                .start_turn(StartTurnRequest {
8491                    thread_id: "thread-turn-context".to_string(),
8492                    message: "hello".to_string(),
8493                    images: Vec::new(),
8494                    provider_override: None,
8495                    model_override: None,
8496                    reasoning_override: None,
8497                    workspace: test_workspace(),
8498                    instructions: InstructionBundle::default(),
8499                    developer_context,
8500                    task_ledger_required: false,
8501                    service_tier_override: None,
8502                })
8503                .await
8504                .unwrap();
8505            tokio::time::timeout(std::time::Duration::from_secs(5), async {
8506                loop {
8507                    let envelope = rx.recv().await.unwrap();
8508                    if envelope.turn_id.as_deref() != Some(&turn_id) {
8509                        continue;
8510                    }
8511                    match envelope.event {
8512                        RoderEvent::TurnCompleted(_) => break,
8513                        RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8514                        _ => {}
8515                    }
8516                }
8517            })
8518            .await
8519            .unwrap();
8520        }
8521
8522        run_turn(
8523            &runtime,
8524            Some("Connected accounts: example-service.".to_string()),
8525        )
8526        .await;
8527        let request = captured.lock().unwrap().clone().unwrap();
8528        assert_eq!(
8529            request.instructions.developer_context.as_deref(),
8530            Some("Connected accounts: example-service.")
8531        );
8532
8533        // The context is per-turn only: the next turn on the same thread
8534        // without a developerContext must not see the previous one.
8535        run_turn(&runtime, None).await;
8536        let request = captured.lock().unwrap().clone().unwrap();
8537        assert_eq!(request.instructions.developer_context, None);
8538    }
8539
8540    #[tokio::test]
8541    async fn tool_search_overrides_route_to_next_inference_request() {
8542        let request = captured_profile_request(RuntimeConfig {
8543            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8544            default_model: "gpt-5.4".to_string(),
8545            tool_search: ToolSearchConfig {
8546                mode: roder_api::inference::ToolSearchMode::Auto,
8547                max_catalog_items: Some(100),
8548                ..ToolSearchConfig::default()
8549            },
8550            provider_tool_search: std::collections::HashMap::from([(
8551                roder_api::catalog::PROVIDER_MOCK.to_string(),
8552                roder_api::inference::ToolSearchConfigOverlay {
8553                    include_skills: Some(false),
8554                    provider_variant: Some(roder_api::inference::ToolSearchProviderVariant::Regex),
8555                    ..Default::default()
8556                },
8557            )]),
8558            model_tool_search: std::collections::HashMap::from([(
8559                "gpt-5.4".to_string(),
8560                roder_api::inference::ToolSearchConfigOverlay {
8561                    mode: Some(roder_api::inference::ToolSearchMode::ProviderNative),
8562                    max_catalog_items: Some(25),
8563                    provider_variant: Some(roder_api::inference::ToolSearchProviderVariant::Bm25),
8564                    ..Default::default()
8565                },
8566            )]),
8567            ..RuntimeConfig::default()
8568        })
8569        .await;
8570
8571        assert_eq!(
8572            request.runtime.tool_search.mode,
8573            roder_api::inference::ToolSearchMode::ProviderNative
8574        );
8575        assert_eq!(request.runtime.tool_search.max_catalog_items, Some(25));
8576        assert_eq!(
8577            request.runtime.tool_search.provider_variant,
8578            roder_api::inference::ToolSearchProviderVariant::Bm25
8579        );
8580    }
8581
8582    #[tokio::test]
8583    async fn context_entrypoint_hints_use_turn_workspace() {
8584        let process_workspace = runtime_test_workspace("entrypoint-process");
8585        let thread_workspace = runtime_test_workspace("entrypoint-thread");
8586        std::fs::create_dir_all(process_workspace.join("src")).unwrap();
8587        std::fs::create_dir_all(thread_workspace.join("src")).unwrap();
8588        std::fs::write(
8589            process_workspace.join("src/sidebar-thread-groups.ts"),
8590            "export const desktopLeak = true;\n",
8591        )
8592        .unwrap();
8593        std::fs::write(
8594            thread_workspace.join("src/voice-plan-feedback.ts"),
8595            "export const voicePlanFeedback = true;\n",
8596        )
8597        .unwrap();
8598
8599        let captured = Arc::new(StdMutex::new(None));
8600        let mut builder = ExtensionRegistryBuilder::new();
8601        builder.inference_engine(Arc::new(CapturingEngine {
8602            request: captured.clone(),
8603        }));
8604        builder.context_planner(Arc::new(roder_context::EntrypointContextPlanner::new(
8605            process_workspace.clone(),
8606        )));
8607        let runtime = Arc::new(
8608            Runtime::new(
8609                builder.build().unwrap(),
8610                RuntimeConfig {
8611                    workspace: Some(process_workspace.display().to_string()),
8612                    ..RuntimeConfig::default()
8613                },
8614            )
8615            .unwrap(),
8616        );
8617        let mut rx = runtime.subscribe_events();
8618
8619        let turn_id = runtime
8620            .start_turn(StartTurnRequest {
8621                thread_id: "thread-workspace-entrypoint".to_string(),
8622                message: "investigate voice plan feedback".to_string(),
8623                images: Vec::new(),
8624                provider_override: None,
8625                model_override: None,
8626                reasoning_override: None,
8627                workspace: thread_workspace.display().to_string(),
8628                instructions: crate::instructions::default_instructions(),
8629                developer_context: None,
8630                task_ledger_required: false,
8631                service_tier_override: None,
8632            })
8633            .await
8634            .unwrap();
8635
8636        tokio::time::timeout(std::time::Duration::from_secs(5), async {
8637            loop {
8638                let envelope = rx.recv().await.unwrap();
8639                if envelope.turn_id.as_deref() != Some(&turn_id) {
8640                    continue;
8641                }
8642                match envelope.event {
8643                    RoderEvent::TurnCompleted(_) => break,
8644                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8645                    _ => {}
8646                }
8647            }
8648        })
8649        .await
8650        .unwrap();
8651
8652        let request = captured.lock().unwrap().clone().expect("captured request");
8653        let transcript_text = request
8654            .transcript
8655            .iter()
8656            .map(|item| match item {
8657                TranscriptItem::UserMessage(message) => message.text.as_str(),
8658                _ => "",
8659            })
8660            .collect::<Vec<_>>()
8661            .join("\n");
8662        assert!(transcript_text.contains("src/voice-plan-feedback.ts"));
8663        assert!(!transcript_text.contains("src/sidebar-thread-groups.ts"));
8664
8665        let _ = std::fs::remove_dir_all(process_workspace);
8666        let _ = std::fs::remove_dir_all(thread_workspace);
8667    }
8668
8669    fn runtime_test_workspace(name: &str) -> std::path::PathBuf {
8670        let path =
8671            std::env::temp_dir().join(format!("roder-runtime-{name}-{}", uuid::Uuid::new_v4()));
8672        let _ = std::fs::remove_dir_all(&path);
8673        std::fs::create_dir_all(&path).unwrap();
8674        path
8675    }
8676
8677    #[tokio::test]
8678    async fn model_profile_user_model_knobs_override_profile_defaults() {
8679        let request = captured_profile_request(RuntimeConfig {
8680            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8681            default_model: "gpt-5.5".to_string(),
8682            reasoning: Some(REASONING_HIGH.to_string()),
8683            auto_compact_token_limit: Some(999),
8684            model_edit_tools: std::collections::HashMap::from([(
8685                "gpt-5.5".to_string(),
8686                EDIT_TOOL_PATCH.to_string(),
8687            )]),
8688            model_parallel_tool_calls: std::collections::HashMap::from([(
8689                "gpt-5.5".to_string(),
8690                true,
8691            )]),
8692            model_profiles: std::collections::HashMap::from([(
8693                "gpt-5.5".to_string(),
8694                test_model_profile("gpt-5.5"),
8695            )]),
8696            ..RuntimeConfig::default()
8697        })
8698        .await;
8699
8700        let tool_names = request
8701            .tools
8702            .iter()
8703            .map(|tool| tool.name.as_str())
8704            .collect::<Vec<_>>();
8705        assert!(tool_names.contains(&"apply_patch"));
8706        assert!(!tool_names.contains(&"edit"));
8707        assert_eq!(request.reasoning.level.as_deref(), Some(REASONING_HIGH));
8708        assert_eq!(request.runtime.parallel_tool_calls, Some(true));
8709        assert_eq!(request.runtime.auto_compact_token_limit, Some(999));
8710    }
8711
8712    #[tokio::test]
8713    async fn runtime_tool_allowlist_filters_advertised_tools() {
8714        let request = captured_profile_request(RuntimeConfig {
8715            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8716            default_model: "gpt-5.5".to_string(),
8717            tool_allowlist: vec!["edit".to_string()],
8718            model_profiles: std::collections::HashMap::from([(
8719                "gpt-5.5".to_string(),
8720                test_model_profile("gpt-5.5"),
8721            )]),
8722            ..RuntimeConfig::default()
8723        })
8724        .await;
8725
8726        let tool_names = request
8727            .tools
8728            .iter()
8729            .map(|tool| tool.name.as_str())
8730            .collect::<Vec<_>>();
8731        assert_eq!(tool_names, vec!["edit"]);
8732    }
8733
8734    /// Builds a store-backed runtime turn for a thread created with the given per-thread
8735    /// overrides and returns the captured inference request.
8736    async fn captured_thread_override_request(
8737        runtime: &Arc<Runtime>,
8738        requests: &Arc<StdMutex<Vec<AgentInferenceRequest>>>,
8739        tool_allowlist: Vec<String>,
8740        developer_instructions: Option<String>,
8741        external_tools: Vec<ToolSpec>,
8742    ) -> AgentInferenceRequest {
8743        let thread_id = runtime
8744            .create_thread_with(CreateThreadRequest {
8745                title: Some("Thread overrides".to_string()),
8746                workspace: test_workspace(),
8747                workspace_id: None,
8748                root_id: None,
8749                provider: None,
8750                model: None,
8751                selection_mode: None,
8752                tool_allowlist,
8753                developer_instructions,
8754                external_tools,
8755                runner: None,
8756            })
8757            .await
8758            .unwrap()
8759            .thread_id;
8760        let mut rx = runtime.subscribe_events();
8761        let turn_id = runtime
8762            .start_turn(StartTurnRequest {
8763                thread_id,
8764                message: "hello".to_string(),
8765                images: Vec::new(),
8766                provider_override: None,
8767                model_override: None,
8768                reasoning_override: None,
8769                workspace: test_workspace(),
8770                instructions: crate::default_instructions(),
8771                developer_context: None,
8772                task_ledger_required: false,
8773                service_tier_override: None,
8774            })
8775            .await
8776            .unwrap();
8777        tokio::time::timeout(std::time::Duration::from_secs(5), async {
8778            loop {
8779                let envelope = rx.recv().await.unwrap();
8780                if envelope.turn_id.as_deref() != Some(&turn_id) {
8781                    continue;
8782                }
8783                match envelope.event {
8784                    RoderEvent::TurnCompleted(_) => break,
8785                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8786                    _ => {}
8787                }
8788            }
8789        })
8790        .await
8791        .unwrap();
8792        requests.lock().unwrap().pop().expect("captured request")
8793    }
8794
8795    #[tokio::test]
8796    async fn thread_tool_allowlist_filters_only_that_thread() {
8797        let requests = Arc::new(StdMutex::new(Vec::new()));
8798        let thread_root =
8799            std::env::temp_dir().join(format!("roder-thread-allowlist-{}", uuid::Uuid::new_v4()));
8800        let mut builder = ExtensionRegistryBuilder::new();
8801        builder.inference_engine(Arc::new(SwitchCaptureEngine {
8802            requests: requests.clone(),
8803        }));
8804        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8805            base_path: thread_root.clone(),
8806        }));
8807        builder.tool_contributor(Arc::new(ProfileToolContributor));
8808        let runtime = Arc::new(
8809            Runtime::new(
8810                builder.build().unwrap(),
8811                RuntimeConfig {
8812                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8813                    default_model: "gpt-5.5".to_string(),
8814                    model_profiles: std::collections::HashMap::from([(
8815                        "gpt-5.5".to_string(),
8816                        test_model_profile("gpt-5.5"),
8817                    )]),
8818                    ..RuntimeConfig::default()
8819                },
8820            )
8821            .unwrap(),
8822        );
8823
8824        let allowlisted = captured_thread_override_request(
8825            &runtime,
8826            &requests,
8827            vec!["edit".to_string()],
8828            None,
8829            Vec::new(),
8830        )
8831        .await;
8832        let unrestricted =
8833            captured_thread_override_request(&runtime, &requests, Vec::new(), None, Vec::new())
8834                .await;
8835
8836        let allowlisted_names = allowlisted
8837            .tools
8838            .iter()
8839            .map(|tool| tool.name.as_str())
8840            .collect::<Vec<_>>();
8841        assert_eq!(allowlisted_names, vec!["edit"]);
8842        let unrestricted_names = unrestricted
8843            .tools
8844            .iter()
8845            .map(|tool| tool.name.as_str())
8846            .collect::<Vec<_>>();
8847        assert!(unrestricted_names.contains(&"edit"));
8848        assert!(unrestricted_names.len() > 1);
8849
8850        let _ = std::fs::remove_dir_all(thread_root);
8851    }
8852
8853    /// Builds a store-backed runtime whose `RuntimeConfig.tool_allowlist` is `["edit"]`.
8854    fn runtime_with_edit_allowlist(
8855        requests: &Arc<StdMutex<Vec<AgentInferenceRequest>>>,
8856        thread_root: &std::path::Path,
8857    ) -> Arc<Runtime> {
8858        let mut builder = ExtensionRegistryBuilder::new();
8859        builder.inference_engine(Arc::new(SwitchCaptureEngine {
8860            requests: requests.clone(),
8861        }));
8862        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8863            base_path: thread_root.to_path_buf(),
8864        }));
8865        builder.tool_contributor(Arc::new(ProfileToolContributor));
8866        Arc::new(
8867            Runtime::new(
8868                builder.build().unwrap(),
8869                RuntimeConfig {
8870                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8871                    default_model: "gpt-5.5".to_string(),
8872                    tool_allowlist: vec!["edit".to_string()],
8873                    model_profiles: std::collections::HashMap::from([(
8874                        "gpt-5.5".to_string(),
8875                        test_model_profile("gpt-5.5"),
8876                    )]),
8877                    ..RuntimeConfig::default()
8878                },
8879            )
8880            .unwrap(),
8881        )
8882    }
8883
8884    #[tokio::test]
8885    async fn runtime_and_thread_allowlists_intersect() {
8886        let requests = Arc::new(StdMutex::new(Vec::new()));
8887        let thread_root = std::env::temp_dir().join(format!(
8888            "roder-allowlist-intersect-{}",
8889            uuid::Uuid::new_v4()
8890        ));
8891        let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8892
8893        let request = captured_thread_override_request(
8894            &runtime,
8895            &requests,
8896            vec!["edit".to_string(), "write_file".to_string()],
8897            None,
8898            Vec::new(),
8899        )
8900        .await;
8901
8902        // The thread allowlist must not re-enable tools the runtime allowlist bans.
8903        let names = request
8904            .tools
8905            .iter()
8906            .map(|tool| tool.name.as_str())
8907            .collect::<Vec<_>>();
8908        assert_eq!(names, vec!["edit"]);
8909
8910        let _ = std::fs::remove_dir_all(thread_root);
8911    }
8912
8913    #[tokio::test]
8914    async fn route_tool_call_denies_tools_outside_allowlists() {
8915        let requests = Arc::new(StdMutex::new(Vec::new()));
8916        let thread_root =
8917            std::env::temp_dir().join(format!("roder-allowlist-dispatch-{}", uuid::Uuid::new_v4()));
8918        let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8919        let thread_id = runtime
8920            .create_thread_with(CreateThreadRequest {
8921                title: Some("Dispatch allowlist".to_string()),
8922                workspace: test_workspace(),
8923                workspace_id: None,
8924                root_id: None,
8925                provider: None,
8926                model: None,
8927                selection_mode: None,
8928                tool_allowlist: vec!["edit".to_string(), "write_file".to_string()],
8929                developer_instructions: None,
8930                external_tools: Vec::new(),
8931                runner: None,
8932            })
8933            .await
8934            .unwrap()
8935            .thread_id;
8936
8937        // write_file is registered and on the thread allowlist but banned by the runtime allowlist.
8938        let result = runtime
8939            .route_tool_call(
8940                &thread_id,
8941                &"turn-allowlist-dispatch".to_string(),
8942                roder_api::inference::ToolCallCompleted {
8943                    id: "call-1".to_string(),
8944                    name: "write_file".to_string(),
8945                    arguments: r#"{"path":"a.txt","content":"hi"}"#.to_string(),
8946                },
8947                None,
8948                None,
8949            )
8950            .await
8951            .unwrap();
8952
8953        assert!(result.is_error);
8954        assert!(
8955            result
8956                .result
8957                .contains("not permitted by the tool allowlist"),
8958            "unexpected result: {}",
8959            result.result
8960        );
8961
8962        let _ = std::fs::remove_dir_all(thread_root);
8963    }
8964
8965    #[tokio::test]
8966    async fn route_tool_calls_denies_agent_swarm_mixed_with_other_tools() {
8967        let requests = Arc::new(StdMutex::new(Vec::new()));
8968        let thread_root =
8969            std::env::temp_dir().join(format!("roder-swarm-exclusive-{}", uuid::Uuid::new_v4()));
8970        let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8971
8972        // A response that mixes agent_swarm with another tool is denied wholesale:
8973        // both calls get an error result with retry guidance, and no tool runs.
8974        let results = runtime
8975            .route_tool_calls(
8976                &"thread-swarm".to_string(),
8977                &"turn-swarm".to_string(),
8978                vec![
8979                    roder_api::inference::ToolCallCompleted {
8980                        id: "swarm-1".to_string(),
8981                        name: "agent_swarm".to_string(),
8982                        arguments: "{}".to_string(),
8983                    },
8984                    roder_api::inference::ToolCallCompleted {
8985                        id: "read-1".to_string(),
8986                        name: "read_file".to_string(),
8987                        arguments: "{}".to_string(),
8988                    },
8989                ],
8990                true,
8991                None,
8992                None,
8993            )
8994            .await
8995            .unwrap();
8996
8997        assert_eq!(results.len(), 2, "every tool_call_id must get a response");
8998        assert_eq!(results[0].id, "swarm-1");
8999        assert_eq!(results[1].id, "read-1");
9000        for result in &results {
9001            assert!(result.is_error);
9002            assert!(
9003                result.result.contains("only tool call"),
9004                "unexpected result: {}",
9005                result.result
9006            );
9007        }
9008
9009        let _ = std::fs::remove_dir_all(thread_root);
9010    }
9011
9012    #[tokio::test]
9013    async fn route_tool_calls_emits_agent_swarm_started_event() {
9014        let requests = Arc::new(StdMutex::new(Vec::new()));
9015        let thread_root =
9016            std::env::temp_dir().join(format!("roder-swarm-started-{}", uuid::Uuid::new_v4()));
9017        let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
9018        let mut events = runtime.subscribe_events();
9019
9020        // A single agent_swarm call (the valid shape) emits AgentSwarmStarted on
9021        // the event bus before dispatch, with the child count parsed from args.
9022        let _ = runtime
9023            .route_tool_calls(
9024                &"thread-swarm".to_string(),
9025                &"turn-swarm".to_string(),
9026                vec![roder_api::inference::ToolCallCompleted {
9027                    id: "swarm-1".to_string(),
9028                    name: "agent_swarm".to_string(),
9029                    arguments: r#"{"description":"x","prompt_template":"Read {{item}}","items":["a.rs","b.rs"]}"#
9030                        .to_string(),
9031                }],
9032                true,
9033                None,
9034                None,
9035            )
9036            .await
9037            .unwrap();
9038
9039        let mut started_child_count = None;
9040        for _ in 0..16 {
9041            let envelope = tokio::time::timeout(std::time::Duration::from_secs(2), events.recv())
9042                .await
9043                .unwrap()
9044                .unwrap();
9045            if let RoderEvent::AgentSwarmStarted(event) = envelope.event {
9046                started_child_count = Some(event.child_count);
9047                assert_eq!(event.tool_id, "swarm-1");
9048                break;
9049            }
9050        }
9051        assert_eq!(started_child_count, Some(2));
9052
9053        let _ = std::fs::remove_dir_all(thread_root);
9054    }
9055
9056    #[test]
9057    fn agent_swarm_child_count_sums_items_and_resumes() {
9058        assert_eq!(
9059            agent_swarm_child_count(r#"{"description":"x","items":["a","b","c"]}"#),
9060            3
9061        );
9062        assert_eq!(
9063            agent_swarm_child_count(
9064                r#"{"description":"x","items":["a"],"resume_agent_ids":{"id1":"continue"}}"#
9065            ),
9066            2
9067        );
9068        // Malformed input is lenient.
9069        assert_eq!(agent_swarm_child_count("not json"), 0);
9070    }
9071
9072    #[test]
9073    fn parse_swarm_counts_reads_summary_with_omitted_buckets() {
9074        let text = "<agent_swarm_result>\n<summary>completed: 2, failed: 1</summary>\n</agent_swarm_result>";
9075        assert_eq!(parse_swarm_counts(text), Some((2, 1, 0)));
9076        let text = "<agent_swarm_result>\n<summary>completed: 0</summary>\n</agent_swarm_result>";
9077        assert_eq!(parse_swarm_counts(text), Some((0, 0, 0)));
9078        // Not a swarm result.
9079        assert_eq!(parse_swarm_counts("just text"), None);
9080    }
9081
9082    /// Signals when inference starts, then waits for `proceed` and fails the stream.
9083    struct SignalledFailureEngine {
9084        started: tokio::sync::mpsc::UnboundedSender<()>,
9085        proceed: Arc<tokio::sync::Notify>,
9086    }
9087
9088    #[async_trait::async_trait]
9089    impl InferenceEngine for SignalledFailureEngine {
9090        fn id(&self) -> String {
9091            roder_api::catalog::PROVIDER_MOCK.to_string()
9092        }
9093
9094        fn capabilities(&self) -> InferenceCapabilities {
9095            InferenceCapabilities::coding_agent_default()
9096        }
9097
9098        async fn list_models(
9099            &self,
9100            _ctx: InferenceProviderContext<'_>,
9101        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
9102            Ok(roder_api::catalog::models_for_provider(
9103                roder_api::catalog::PROVIDER_MOCK,
9104                true,
9105            ))
9106        }
9107
9108        async fn stream_turn(
9109            &self,
9110            _ctx: InferenceTurnContext<'_>,
9111            _request: AgentInferenceRequest,
9112        ) -> anyhow::Result<InferenceEventStream> {
9113            let _ = self.started.send(());
9114            self.proceed.notified().await;
9115            anyhow::bail!("engine failed mid-turn")
9116        }
9117    }
9118
9119    #[tokio::test]
9120    async fn failed_turn_sweeps_pending_external_tool_calls() {
9121        let (started_tx, mut started_rx) = tokio::sync::mpsc::unbounded_channel();
9122        let proceed = Arc::new(tokio::sync::Notify::new());
9123        let mut builder = ExtensionRegistryBuilder::new();
9124        builder.inference_engine(Arc::new(SignalledFailureEngine {
9125            started: started_tx,
9126            proceed: proceed.clone(),
9127        }));
9128        let runtime =
9129            Arc::new(Runtime::new(builder.build().unwrap(), RuntimeConfig::default()).unwrap());
9130        let mut rx = runtime.subscribe_events();
9131        let turn_id = runtime
9132            .start_turn(StartTurnRequest {
9133                thread_id: "thread-sweep".to_string(),
9134                message: "go".to_string(),
9135                images: Vec::new(),
9136                provider_override: None,
9137                model_override: None,
9138                reasoning_override: None,
9139                workspace: test_workspace(),
9140                instructions: InstructionBundle {
9141                    system: None,
9142                    developer: None,
9143                    developer_context: None,
9144                },
9145                developer_context: None,
9146                task_ledger_required: false,
9147                service_tier_override: None,
9148            })
9149            .await
9150            .unwrap();
9151        tokio::time::timeout(std::time::Duration::from_secs(5), started_rx.recv())
9152            .await
9153            .unwrap()
9154            .unwrap();
9155
9156        let (tx, _pending_rx) = oneshot::channel();
9157        runtime.pending_external_tool_calls.lock().await.insert(
9158            "exttool-sweep-test".to_string(),
9159            PendingExternalToolCall {
9160                thread_id: "thread-sweep".to_string(),
9161                turn_id: turn_id.clone(),
9162                tool_id: "call-1".to_string(),
9163                tool_name: "acme_lookup".to_string(),
9164                tx,
9165            },
9166        );
9167        proceed.notify_one();
9168
9169        let outcome = tokio::time::timeout(std::time::Duration::from_secs(5), async {
9170            loop {
9171                let envelope = rx.recv().await.unwrap();
9172                if let RoderEvent::ExternalToolCallResolved(event) = envelope.event
9173                    && event.request_id == "exttool-sweep-test"
9174                {
9175                    break event.outcome;
9176                }
9177            }
9178        })
9179        .await
9180        .expect("turn failure must resolve pending external tool calls");
9181        assert_eq!(outcome, ExternalToolCallOutcome::Cancelled);
9182        assert!(runtime.pending_external_tool_calls.lock().await.is_empty());
9183    }
9184
9185    #[tokio::test]
9186    async fn thread_developer_instructions_layer_under_harness_prompt() {
9187        let requests = Arc::new(StdMutex::new(Vec::new()));
9188        let thread_root = std::env::temp_dir().join(format!(
9189            "roder-thread-instructions-{}",
9190            uuid::Uuid::new_v4()
9191        ));
9192        let mut builder = ExtensionRegistryBuilder::new();
9193        builder.inference_engine(Arc::new(SwitchCaptureEngine {
9194            requests: requests.clone(),
9195        }));
9196        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
9197            base_path: thread_root.clone(),
9198        }));
9199        builder.tool_contributor(Arc::new(ProfileToolContributor));
9200        let runtime =
9201            Arc::new(Runtime::new(builder.build().unwrap(), RuntimeConfig::default()).unwrap());
9202
9203        let request = captured_thread_override_request(
9204            &runtime,
9205            &requests,
9206            Vec::new(),
9207            Some("You are embedded in a host app.".to_string()),
9208            Vec::new(),
9209        )
9210        .await;
9211
9212        let system = request.instructions.system.expect("system instructions");
9213        assert!(system.starts_with("You are Roder"));
9214        let developer = request
9215            .instructions
9216            .developer
9217            .expect("developer instructions");
9218        assert!(developer.starts_with("You are embedded in a host app."));
9219
9220        let plain =
9221            captured_thread_override_request(&runtime, &requests, Vec::new(), None, Vec::new())
9222                .await;
9223        assert_eq!(plain.instructions.developer, None);
9224
9225        let _ = std::fs::remove_dir_all(thread_root);
9226    }
9227
9228    #[tokio::test]
9229    async fn thread_external_tools_are_advertised_and_shadow_builtins() {
9230        let requests = Arc::new(StdMutex::new(Vec::new()));
9231        let thread_root = std::env::temp_dir().join(format!(
9232            "roder-thread-external-tools-{}",
9233            uuid::Uuid::new_v4()
9234        ));
9235        let mut builder = ExtensionRegistryBuilder::new();
9236        builder.inference_engine(Arc::new(SwitchCaptureEngine {
9237            requests: requests.clone(),
9238        }));
9239        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
9240            base_path: thread_root.clone(),
9241        }));
9242        builder.tool_contributor(Arc::new(ProfileToolContributor));
9243        let runtime =
9244            Arc::new(Runtime::new(builder.build().unwrap(), RuntimeConfig::default()).unwrap());
9245
9246        let external_tools = vec![
9247            ToolSpec {
9248                name: "acme_lookup".to_string(),
9249                description: "Look up Acme workspace state.".to_string(),
9250                parameters: serde_json::json!({
9251                    "type": "object",
9252                    "properties": { "query": { "type": "string" } },
9253                    "required": ["query"]
9254                }),
9255            },
9256            ToolSpec {
9257                name: "edit".to_string(),
9258                description: "Host-managed edit.".to_string(),
9259                parameters: serde_json::json!({ "type": "object" }),
9260            },
9261        ];
9262        let request =
9263            captured_thread_override_request(&runtime, &requests, Vec::new(), None, external_tools)
9264                .await;
9265
9266        let acme = request
9267            .tools
9268            .iter()
9269            .find(|tool| tool.name == "acme_lookup")
9270            .expect("external tool advertised");
9271        assert_eq!(acme.description, "Look up Acme workspace state.");
9272        assert_eq!(acme.parameters["required"][0], "query");
9273        let edits = request
9274            .tools
9275            .iter()
9276            .filter(|tool| tool.name == "edit")
9277            .collect::<Vec<_>>();
9278        assert_eq!(edits.len(), 1, "external edit shadows the builtin");
9279        assert_eq!(edits[0].description, "Host-managed edit.");
9280
9281        let plain =
9282            captured_thread_override_request(&runtime, &requests, Vec::new(), None, Vec::new())
9283                .await;
9284        assert!(plain.tools.iter().all(|tool| tool.name != "acme_lookup"));
9285        let plain_edit = plain
9286            .tools
9287            .iter()
9288            .find(|tool| tool.name == "edit")
9289            .expect("builtin edit advertised on plain thread");
9290        assert_eq!(plain_edit.description, "edit test tool");
9291
9292        let _ = std::fs::remove_dir_all(thread_root);
9293    }
9294
9295    #[tokio::test]
9296    async fn model_switch_injects_summary_and_records_profile_segments() {
9297        let requests = Arc::new(StdMutex::new(Vec::new()));
9298        let thread_root = std::env::temp_dir().join(format!(
9299            "roder-model-switch-thread-{}",
9300            uuid::Uuid::new_v4()
9301        ));
9302        let mut builder = ExtensionRegistryBuilder::new();
9303        builder.inference_engine(Arc::new(SwitchCaptureEngine {
9304            requests: requests.clone(),
9305        }));
9306        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
9307            base_path: thread_root.clone(),
9308        }));
9309        builder.tool_contributor(Arc::new(ProfileToolContributor));
9310        let mut claude_profile = test_model_profile("claude-haiku-4-5-20251001");
9311        claude_profile.provider_family = ProviderFamily::Anthropic;
9312        claude_profile.edit_tool = Some(EDIT_TOOL_EDIT.to_string());
9313        let runtime = Arc::new(
9314            Runtime::new(
9315                builder.build().unwrap(),
9316                RuntimeConfig {
9317                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9318                    default_model: "gpt-5.5".to_string(),
9319                    model_profiles: std::collections::HashMap::from([
9320                        ("gpt-5.5".to_string(), test_model_profile("gpt-5.5")),
9321                        ("claude-haiku-4-5-20251001".to_string(), claude_profile),
9322                    ]),
9323                    ..RuntimeConfig::default()
9324                },
9325            )
9326            .unwrap(),
9327        );
9328        let thread_id = runtime
9329            .create_thread_with(CreateThreadRequest {
9330                title: Some("Model switch".to_string()),
9331                workspace: test_workspace(),
9332                workspace_id: None,
9333                root_id: None,
9334                provider: None,
9335                model: None,
9336                selection_mode: None,
9337                tool_allowlist: Vec::new(),
9338                developer_instructions: None,
9339                external_tools: Vec::new(),
9340                runner: None,
9341            })
9342            .await
9343            .unwrap()
9344            .thread_id;
9345        let mut rx = runtime.subscribe_events();
9346        for (message, model_override) in [
9347            ("first turn", None),
9348            ("second turn", Some("claude-haiku-4-5-20251001".to_string())),
9349        ] {
9350            let turn_id = runtime
9351                .start_turn(StartTurnRequest {
9352                    thread_id: thread_id.clone(),
9353                    message: message.to_string(),
9354                    images: Vec::new(),
9355                    provider_override: None,
9356                    model_override,
9357                    reasoning_override: None,
9358                    workspace: test_workspace(),
9359                    instructions: InstructionBundle::default(),
9360                    developer_context: None,
9361                    task_ledger_required: false,
9362                    service_tier_override: None,
9363                })
9364                .await
9365                .unwrap();
9366            tokio::time::timeout(std::time::Duration::from_secs(5), async {
9367                loop {
9368                    let envelope = rx.recv().await.unwrap();
9369                    if envelope.turn_id.as_deref() != Some(&turn_id) {
9370                        continue;
9371                    }
9372                    match envelope.event {
9373                        RoderEvent::TurnCompleted(_) => break,
9374                        RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9375                        _ => {}
9376                    }
9377                }
9378            })
9379            .await
9380            .unwrap();
9381        }
9382
9383        let captured = requests.lock().unwrap().clone();
9384        assert_eq!(captured.len(), 2);
9385        assert!(captured[1].transcript.iter().any(|item| {
9386            matches!(
9387                item,
9388                TranscriptItem::UserMessage(message)
9389                    if message.text.starts_with(MODEL_SWITCH_SUMMARY_PREFIX)
9390                        && message.text.contains("previous profile mock/gpt-5.5")
9391                        && message.text.contains("Current profile mock/claude-haiku-4-5-20251001")
9392                        && message.text.contains("Available tools now:")
9393            )
9394        }));
9395
9396        let snapshot = runtime
9397            .thread_store
9398            .as_ref()
9399            .unwrap()
9400            .load_thread(&thread_id)
9401            .await
9402            .unwrap()
9403            .unwrap();
9404        let trace_segments = snapshot
9405            .turns
9406            .iter()
9407            .flat_map(|turn| &turn.items)
9408            .filter(|item| {
9409                matches!(
9410                    item,
9411                    TranscriptItem::ProviderMetadata(value)
9412                        if value.get("kind").and_then(serde_json::Value::as_str)
9413                            == Some(MODEL_PROFILE_TRACE_KIND)
9414                            && value.get("segment").and_then(serde_json::Value::as_str)
9415                                == Some("assistant")
9416                )
9417            })
9418            .count();
9419        assert!(trace_segments >= 2);
9420        let _ = std::fs::remove_dir_all(thread_root);
9421    }
9422
9423    struct CountingTaskTool {
9424        calls: Arc<StdMutex<u32>>,
9425    }
9426
9427    #[async_trait::async_trait]
9428    impl ToolExecutor for CountingTaskTool {
9429        fn spec(&self) -> ToolSpec {
9430            ToolSpec {
9431                name: "task".to_string(),
9432                description: "Dispatch a test subagent.".to_string(),
9433                parameters: serde_json::json!({
9434                    "type": "object",
9435                    "properties": {
9436                        "description": { "type": "string" },
9437                        "prompt": { "type": "string" },
9438                        "parent_deadline_seconds": { "type": "integer" }
9439                    },
9440                    "required": ["description", "prompt"],
9441                    "additionalProperties": false
9442                }),
9443            }
9444        }
9445
9446        async fn execute(
9447            &self,
9448            _ctx: ToolExecutionContext,
9449            call: ToolCall,
9450        ) -> anyhow::Result<ToolResult> {
9451            *self.calls.lock().unwrap() += 1;
9452            Ok(ToolResult {
9453                id: call.id,
9454                name: call.name,
9455                text: "started child".to_string(),
9456                data: serde_json::json!({}),
9457                is_error: false,
9458            })
9459        }
9460    }
9461
9462    #[tokio::test]
9463    async fn runtime_profile_reaches_inference_request_and_turn_metadata() {
9464        let captured = Arc::new(StdMutex::new(None));
9465        let mut builder = ExtensionRegistryBuilder::new();
9466        builder.inference_engine(Arc::new(CapturingEngine {
9467            request: captured.clone(),
9468        }));
9469        let runtime = Arc::new(
9470            Runtime::new(
9471                builder.build().unwrap(),
9472                RuntimeConfig {
9473                    runtime_profile: RuntimeProfile::NonInteractive,
9474                    ..RuntimeConfig::default()
9475                },
9476            )
9477            .unwrap(),
9478        );
9479        let mut rx = runtime.subscribe_events();
9480        let turn_id = runtime
9481            .start_turn(StartTurnRequest {
9482                thread_id: "thread-profile".to_string(),
9483                message: "work unattended".to_string(),
9484                images: Vec::new(),
9485                provider_override: None,
9486                model_override: None,
9487                reasoning_override: None,
9488                workspace: test_workspace(),
9489                instructions: InstructionBundle {
9490                    system: None,
9491                    developer: Some("base developer".to_string()),
9492                    developer_context: None,
9493                },
9494                developer_context: None,
9495                task_ledger_required: false,
9496                service_tier_override: None,
9497            })
9498            .await
9499            .unwrap();
9500
9501        let mut observed_profile = None;
9502        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9503            loop {
9504                let envelope = rx.recv().await.unwrap();
9505                if envelope.turn_id.as_deref() != Some(&turn_id) {
9506                    continue;
9507                }
9508                match envelope.event {
9509                    RoderEvent::TurnStarted(event) => {
9510                        observed_profile = Some(event.runtime_profile);
9511                    }
9512                    RoderEvent::TurnCompleted(_) => break,
9513                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9514                    _ => {}
9515                }
9516            }
9517        })
9518        .await
9519        .unwrap();
9520
9521        assert_eq!(observed_profile, Some(RuntimeProfile::NonInteractive));
9522        let request = captured.lock().unwrap().clone().unwrap();
9523        assert_eq!(request.runtime.profile, RuntimeProfile::NonInteractive);
9524        let developer = request.instructions.developer.unwrap();
9525        assert!(developer.contains("base developer"));
9526        assert!(developer.contains("non-interactive profile"));
9527    }
9528
9529    #[tokio::test]
9530    async fn global_policy_mode_changes_do_not_create_runtime_thread_directory() {
9531        let workspace = runtime_test_workspace("global-policy-mode");
9532        let thread_root = workspace.join("threads");
9533        let mut builder = ExtensionRegistryBuilder::new();
9534        builder.inference_engine(Arc::new(FakeInferenceEngine));
9535        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
9536            base_path: thread_root.clone(),
9537        }));
9538        let runtime = Runtime::new(
9539            builder.build().unwrap(),
9540            RuntimeConfig {
9541                workspace: Some(workspace.display().to_string()),
9542                ..Default::default()
9543            },
9544        )
9545        .unwrap();
9546
9547        runtime
9548            .set_policy_mode(PolicyMode::AcceptAll, Some("test".to_string()))
9549            .await
9550            .unwrap();
9551
9552        assert!(!thread_root.join("runtime").exists());
9553        let _ = std::fs::remove_dir_all(workspace);
9554    }
9555
9556    #[tokio::test]
9557    async fn task_ledger_enforcement_injects_eval_reminder_before_work() {
9558        let captured = Arc::new(StdMutex::new(None));
9559        let mut builder = ExtensionRegistryBuilder::new();
9560        builder.inference_engine(Arc::new(CapturingEngine {
9561            request: captured.clone(),
9562        }));
9563        builder.tool_contributor(Arc::new(
9564            roder_ext_task_ledger::TaskLedgerToolContributor::default(),
9565        ));
9566        let runtime = Arc::new(
9567            Runtime::new(
9568                builder.build().unwrap(),
9569                RuntimeConfig {
9570                    runtime_profile: RuntimeProfile::Eval,
9571                    policy_mode: PolicyMode::Bypass,
9572                    agent_swarm_mode: false,
9573                    ultra_mode: false,
9574                    ..RuntimeConfig::default()
9575                },
9576            )
9577            .unwrap(),
9578        );
9579        let mut rx = runtime.subscribe_events();
9580        let turn_id = runtime
9581            .start_turn(StartTurnRequest {
9582                thread_id: "thread-ledger".to_string(),
9583                message: "decomposed work".to_string(),
9584                images: Vec::new(),
9585                provider_override: None,
9586                model_override: None,
9587                reasoning_override: None,
9588                workspace: test_workspace(),
9589                instructions: InstructionBundle::default(),
9590                developer_context: None,
9591                task_ledger_required: true,
9592                service_tier_override: None,
9593            })
9594            .await
9595            .unwrap();
9596
9597        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9598            loop {
9599                let envelope = rx.recv().await.unwrap();
9600                if envelope.turn_id.as_deref() == Some(&turn_id)
9601                    && matches!(envelope.event, RoderEvent::TurnCompleted(_))
9602                {
9603                    break;
9604                }
9605            }
9606        })
9607        .await
9608        .unwrap();
9609
9610        let request = captured.lock().unwrap().clone().unwrap();
9611        let developer = request.instructions.developer.unwrap();
9612        assert!(developer.contains("Task Ledger Required"));
9613        assert!(developer.contains("task_ledger.update"));
9614        let tool_names: Vec<_> = request
9615            .tools
9616            .iter()
9617            .map(|tool| tool.name.as_str())
9618            .collect();
9619        assert!(
9620            tool_names.contains(&TASK_LEDGER_TOOL_NAME),
9621            "tool names: {tool_names:?}"
9622        );
9623        assert_eq!(
9624            request.tool_choice,
9625            ToolChoice::Specific(TASK_LEDGER_TOOL_NAME.to_string())
9626        );
9627        assert_eq!(request.tools.len(), 1);
9628        assert_eq!(request.tools[0].name, TASK_LEDGER_TOOL_NAME);
9629    }
9630
9631    #[tokio::test]
9632    async fn eval_task_ledger_blocks_final_answer_until_open_items_are_completed() {
9633        let requests = Arc::new(StdMutex::new(Vec::new()));
9634        let mut builder = ExtensionRegistryBuilder::new();
9635        builder.inference_engine(Arc::new(TaskLedgerCompletionGateEngine {
9636            calls: StdMutex::new(0),
9637            requests: requests.clone(),
9638        }));
9639        builder.tool_contributor(Arc::new(
9640            roder_ext_task_ledger::TaskLedgerToolContributor::default(),
9641        ));
9642        let runtime = Arc::new(
9643            Runtime::new(
9644                builder.build().unwrap(),
9645                RuntimeConfig {
9646                    runtime_profile: RuntimeProfile::Eval,
9647                    policy_mode: PolicyMode::Bypass,
9648                    agent_swarm_mode: false,
9649                    ultra_mode: false,
9650                    ..RuntimeConfig::default()
9651                },
9652            )
9653            .unwrap(),
9654        );
9655        let mut rx = runtime.subscribe_events();
9656        let turn_id = runtime
9657            .start_turn(StartTurnRequest {
9658                thread_id: "thread-ledger-completion".to_string(),
9659                message: "write the answer file".to_string(),
9660                images: Vec::new(),
9661                provider_override: None,
9662                model_override: None,
9663                reasoning_override: None,
9664                workspace: test_workspace(),
9665                instructions: InstructionBundle::default(),
9666                developer_context: None,
9667                task_ledger_required: true,
9668                service_tier_override: None,
9669            })
9670            .await
9671            .unwrap();
9672
9673        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9674            loop {
9675                let envelope = rx.recv().await.unwrap();
9676                if envelope.turn_id.as_deref() != Some(&turn_id) {
9677                    continue;
9678                }
9679                match envelope.event {
9680                    RoderEvent::TurnCompleted(_) => break,
9681                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9682                    _ => {}
9683                }
9684            }
9685        })
9686        .await
9687        .unwrap();
9688
9689        let requests = requests.lock().unwrap().clone();
9690        assert_eq!(requests.len(), 4);
9691        assert!(requests[2].transcript.iter().any(|item| {
9692            matches!(
9693                item,
9694                TranscriptItem::UserMessage(message)
9695                    if message.text.contains("Task Ledger Completion Required")
9696                        && message.text.contains("Write /app/result.txt")
9697            )
9698        }));
9699        assert!(requests[3].transcript.iter().any(|item| {
9700            matches!(
9701                item,
9702                TranscriptItem::ToolResult(result)
9703                    if result.name.as_deref() == Some(TASK_LEDGER_TOOL_NAME)
9704                        && result.result.contains("Task ledger: 2/2 completed")
9705            )
9706        }));
9707    }
9708
9709    #[tokio::test]
9710    async fn eval_task_ledger_checkpoint_requests_scoreable_file_before_final_reserve() {
9711        let requests = Arc::new(StdMutex::new(Vec::new()));
9712        let mut builder = ExtensionRegistryBuilder::new();
9713        builder.inference_engine(Arc::new(TaskLedgerCompletionGateEngine {
9714            calls: StdMutex::new(0),
9715            requests: requests.clone(),
9716        }));
9717        builder.tool_contributor(Arc::new(
9718            roder_ext_task_ledger::TaskLedgerToolContributor::default(),
9719        ));
9720        let runtime = Arc::new(
9721            Runtime::new(
9722                builder.build().unwrap(),
9723                RuntimeConfig {
9724                    runtime_profile: RuntimeProfile::Eval,
9725                    policy_mode: PolicyMode::Bypass,
9726                    agent_swarm_mode: false,
9727                    ultra_mode: false,
9728                    turn_deadline_seconds: Some(120),
9729                    ..RuntimeConfig::default()
9730                },
9731            )
9732            .unwrap(),
9733        );
9734        let mut rx = runtime.subscribe_events();
9735        let turn_id = runtime
9736            .start_turn(StartTurnRequest {
9737                thread_id: "thread-ledger-checkpoint".to_string(),
9738                message: "write the answer file".to_string(),
9739                images: Vec::new(),
9740                provider_override: None,
9741                model_override: None,
9742                reasoning_override: None,
9743                workspace: test_workspace(),
9744                instructions: InstructionBundle::default(),
9745                developer_context: None,
9746                task_ledger_required: true,
9747                service_tier_override: None,
9748            })
9749            .await
9750            .unwrap();
9751
9752        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9753            loop {
9754                let envelope = rx.recv().await.unwrap();
9755                if envelope.turn_id.as_deref() != Some(&turn_id) {
9756                    continue;
9757                }
9758                match envelope.event {
9759                    RoderEvent::TurnCompleted(_) => break,
9760                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9761                    _ => {}
9762                }
9763            }
9764        })
9765        .await
9766        .unwrap();
9767
9768        let requests = requests.lock().unwrap().clone();
9769        assert!(requests.len() >= 2);
9770        assert!(requests[1].transcript.iter().any(|item| {
9771            matches!(
9772                item,
9773                TranscriptItem::UserMessage(message)
9774                    if message.text.contains("Scoreable Output Checkpoint")
9775                        && message.text.contains("ensure the required output file(s) exist")
9776                        && message.text.contains("Write /app/result.txt")
9777            )
9778        }));
9779    }
9780
9781    #[test]
9782    fn deadline_task_ledger_prompt_preserves_scoreable_work_instruction() {
9783        let prompt = task_ledger_deadline_completion_prompt(
9784            12,
9785            30,
9786            "Task Ledger Completion Required: write /app/result.txt, then call task_ledger.update",
9787        );
9788
9789        assert!(prompt.contains("12 seconds remain"));
9790        assert!(prompt.contains("create or update the required scoreable output files"));
9791        assert!(prompt.contains("write /app/result.txt"));
9792        assert!(prompt.contains(TASK_LEDGER_TOOL_NAME));
9793    }
9794
9795    #[test]
9796    fn scoreable_checkpoint_prompt_preserves_provisional_file_instruction() {
9797        let prompt = task_ledger_scoreable_checkpoint_prompt(
9798            120,
9799            "Task Ledger Completion Required: write /app/result.txt, then call task_ledger.update",
9800        );
9801
9802        assert!(prompt.contains("120 seconds remain"));
9803        assert!(prompt.contains("best evidence-backed answer"));
9804        assert!(prompt.contains("even if provisional"));
9805        assert!(prompt.contains("preserve that candidate"));
9806        assert!(prompt.contains("partial-coverage"));
9807        assert!(prompt.contains("write /app/result.txt"));
9808        assert!(prompt.contains(TASK_LEDGER_TOOL_NAME));
9809    }
9810
9811    #[test]
9812    fn open_task_ledger_moves_inference_timeout_to_scoreable_checkpoint() {
9813        let deadline = Some(OffsetDateTime::now_utc() + Duration::seconds(870));
9814        let transcript = vec![TranscriptItem::ToolResult(ToolResultRecord {
9815            id: "ledger-open".to_string(),
9816            name: Some(TASK_LEDGER_TOOL_NAME.to_string()),
9817            result: "Task ledger: 0/1 completed\n- pending: Write /app/result.txt [write]"
9818                .to_string(),
9819            display_payload: None,
9820            is_error: false,
9821        })];
9822
9823        let (_, action) = inference_timeout_deadline(
9824            deadline,
9825            RuntimeProfile::Eval,
9826            true,
9827            30,
9828            false,
9829            0,
9830            &transcript,
9831        )
9832        .unwrap();
9833
9834        assert_eq!(action, InferenceTimeoutAction::ScoreableCheckpoint);
9835    }
9836
9837    #[tokio::test]
9838    async fn verification_gate_forces_eval_code_changes_through_review() {
9839        let mut builder = ExtensionRegistryBuilder::new();
9840        builder.inference_engine(Arc::new(VerificationGateEngine {
9841            calls: StdMutex::new(0),
9842        }));
9843        builder.tool_contributor(Arc::new(WriteFileContributor));
9844        builder.tool_contributor(Arc::new(
9845            roder_ext_verification::VerificationToolContributor,
9846        ));
9847        let runtime = Arc::new(
9848            Runtime::new(
9849                builder.build().unwrap(),
9850                RuntimeConfig {
9851                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9852                    default_model: "mock".to_string(),
9853                    runtime_profile: RuntimeProfile::Eval,
9854                    policy_mode: PolicyMode::Bypass,
9855                    agent_swarm_mode: false,
9856                    ultra_mode: false,
9857                    ..RuntimeConfig::default()
9858                },
9859            )
9860            .unwrap(),
9861        );
9862        let mut rx = runtime.subscribe_events();
9863        let turn_id = runtime
9864            .start_turn(StartTurnRequest {
9865                thread_id: "thread-verification".to_string(),
9866                message: "write code".to_string(),
9867                images: Vec::new(),
9868                provider_override: None,
9869                model_override: None,
9870                reasoning_override: None,
9871                workspace: test_workspace(),
9872                instructions: InstructionBundle::default(),
9873                developer_context: None,
9874                task_ledger_required: false,
9875                service_tier_override: None,
9876            })
9877            .await
9878            .unwrap();
9879
9880        let mut saw_required = false;
9881        let mut saw_completed = false;
9882        let mut final_text = String::new();
9883        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9884            loop {
9885                let envelope = rx.recv().await.unwrap();
9886                if envelope.turn_id.as_deref() != Some(&turn_id) {
9887                    continue;
9888                }
9889                match envelope.event {
9890                    RoderEvent::VerificationRequired(event) => {
9891                        saw_required = true;
9892                        assert_eq!(event.changed_files, vec!["src/lib.rs"]);
9893                    }
9894                    RoderEvent::VerificationCompleted(event) => {
9895                        saw_completed = true;
9896                        assert!(event.passed);
9897                    }
9898                    RoderEvent::InferenceEventReceived(event) => {
9899                        if let InferenceEvent::MessageDelta(delta) = event.event {
9900                            final_text.push_str(&delta.text);
9901                        }
9902                    }
9903                    RoderEvent::TurnCompleted(_) => break,
9904                    _ => {}
9905                }
9906            }
9907        })
9908        .await
9909        .unwrap();
9910
9911        assert!(saw_required);
9912        assert!(saw_completed);
9913        assert!(final_text.contains("verified final"));
9914    }
9915
9916    #[tokio::test]
9917    async fn speed_policy_changes_reasoning_across_eval_model_calls_without_model_switch() {
9918        let requests = Arc::new(StdMutex::new(Vec::new()));
9919        let mut builder = ExtensionRegistryBuilder::new();
9920        builder.inference_engine(Arc::new(SpeedPolicyEngine {
9921            calls: StdMutex::new(0),
9922            requests: requests.clone(),
9923        }));
9924        builder.tool_contributor(Arc::new(WriteFileContributor));
9925        builder.tool_contributor(Arc::new(
9926            roder_ext_verification::VerificationToolContributor,
9927        ));
9928        let runtime = Arc::new(
9929            Runtime::new(
9930                builder.build().unwrap(),
9931                RuntimeConfig {
9932                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9933                    default_model: "gpt-5.5".to_string(),
9934                    runtime_profile: RuntimeProfile::Eval,
9935                    policy_mode: PolicyMode::Bypass,
9936                    agent_swarm_mode: false,
9937                    ultra_mode: false,
9938                    ..RuntimeConfig::default()
9939                },
9940            )
9941            .unwrap(),
9942        );
9943        let mut rx = runtime.subscribe_events();
9944        let turn_id = runtime
9945            .start_turn(StartTurnRequest {
9946                thread_id: "thread-speed-policy".to_string(),
9947                message: "write code".to_string(),
9948                images: Vec::new(),
9949                provider_override: None,
9950                model_override: None,
9951                reasoning_override: None,
9952                workspace: test_workspace(),
9953                instructions: InstructionBundle::default(),
9954                developer_context: None,
9955                task_ledger_required: false,
9956                service_tier_override: None,
9957            })
9958            .await
9959            .unwrap();
9960
9961        let mut saw_speed_policy_event = false;
9962        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9963            loop {
9964                let envelope = rx.recv().await.unwrap();
9965                if envelope.turn_id.as_deref() != Some(&turn_id) {
9966                    continue;
9967                }
9968                match envelope.event {
9969                    RoderEvent::InferenceStarted(event) => {
9970                        if event.speed_policy.is_some() {
9971                            saw_speed_policy_event = true;
9972                        }
9973                    }
9974                    RoderEvent::TurnCompleted(_) => break,
9975                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9976                    _ => {}
9977                }
9978            }
9979        })
9980        .await
9981        .unwrap();
9982
9983        let requests = requests.lock().unwrap().clone();
9984        assert!(saw_speed_policy_event);
9985        assert!(requests.len() >= 4);
9986        assert!(requests.iter().all(|request| {
9987            request.model.provider == roder_api::catalog::PROVIDER_MOCK
9988                && request.model.model == "gpt-5.5"
9989        }));
9990        assert_eq!(
9991            requests[0].runtime.speed_policy.as_ref().map(|d| d.phase),
9992            Some(roder_api::inference::SpeedPolicyPhase::Orientation)
9993        );
9994        assert_eq!(requests[0].reasoning.level.as_deref(), Some(REASONING_HIGH));
9995        assert_eq!(
9996            requests[1].runtime.speed_policy.as_ref().map(|d| d.phase),
9997            Some(roder_api::inference::SpeedPolicyPhase::Execution)
9998        );
9999        assert_eq!(requests[1].reasoning.level.as_deref(), Some(REASONING_LOW));
10000        assert_eq!(
10001            requests[2].runtime.speed_policy.as_ref().map(|d| d.phase),
10002            Some(roder_api::inference::SpeedPolicyPhase::Verification)
10003        );
10004        assert_eq!(requests[2].reasoning.level.as_deref(), Some(REASONING_HIGH));
10005        assert_eq!(
10006            requests[2]
10007                .metadata
10008                .pointer("/speedPolicy/phase")
10009                .and_then(serde_json::Value::as_str),
10010            Some("verification")
10011        );
10012    }
10013
10014    #[tokio::test]
10015    async fn deadline_turn_timeout_emits_partial_result_and_clears_active_turn() {
10016        let mut builder = ExtensionRegistryBuilder::new();
10017        builder.inference_engine(Arc::new(DeadlineEngine));
10018        let runtime = Arc::new(
10019            Runtime::new(
10020                builder.build().unwrap(),
10021                RuntimeConfig {
10022                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
10023                    default_model: "mock".to_string(),
10024                    runtime_profile: RuntimeProfile::Eval,
10025                    turn_deadline_seconds: Some(1),
10026                    ..RuntimeConfig::default()
10027                },
10028            )
10029            .unwrap(),
10030        );
10031        let mut rx = runtime.subscribe_events();
10032        let turn_id = runtime
10033            .start_turn(StartTurnRequest {
10034                thread_id: "thread-deadline".to_string(),
10035                message: "slow work".to_string(),
10036                images: Vec::new(),
10037                provider_override: None,
10038                model_override: None,
10039                reasoning_override: None,
10040                workspace: test_workspace(),
10041                instructions: InstructionBundle::default(),
10042                developer_context: None,
10043                task_ledger_required: false,
10044                service_tier_override: None,
10045            })
10046            .await
10047            .unwrap();
10048
10049        let mut saw_partial = false;
10050        let mut saw_deadline = false;
10051        let mut failed_kind = None;
10052        tokio::time::timeout(std::time::Duration::from_secs(5), async {
10053            loop {
10054                let envelope = rx.recv().await.unwrap();
10055                if envelope.turn_id.as_deref() != Some(&turn_id) {
10056                    continue;
10057                }
10058                match envelope.event {
10059                    RoderEvent::TurnPartialResult(event) => {
10060                        saw_partial = event.summary.contains("partial turn state");
10061                    }
10062                    RoderEvent::TurnDeadlineExceeded(event) => {
10063                        saw_deadline = event.partial_result.contains("transcript items");
10064                    }
10065                    RoderEvent::TurnFailed(event) => {
10066                        failed_kind = event.error_kind;
10067                        break;
10068                    }
10069                    _ => {}
10070                }
10071            }
10072        })
10073        .await
10074        .unwrap();
10075
10076        for _ in 0..20 {
10077            if !runtime.active_turns.read().await.contains_key(&turn_id) {
10078                break;
10079            }
10080            tokio::time::sleep(std::time::Duration::from_millis(10)).await;
10081        }
10082        assert!(saw_partial);
10083        assert!(saw_deadline);
10084        assert_eq!(failed_kind.as_deref(), Some("deadline_timeout"));
10085        assert!(!runtime.active_turns.read().await.contains_key(&turn_id));
10086    }
10087
10088    #[tokio::test]
10089    async fn deadline_skips_subagent_task_when_remaining_budget_is_too_low() {
10090        let calls = Arc::new(StdMutex::new(0));
10091        let mut builder = ExtensionRegistryBuilder::new();
10092        builder.inference_engine(Arc::new(CapturingEngine {
10093            request: Arc::new(StdMutex::new(None)),
10094        }));
10095        let task_tool = Arc::new(CountingTaskTool {
10096            calls: calls.clone(),
10097        });
10098        builder.tool_contributor(Arc::new(TestToolContributor { tool: task_tool }));
10099        let runtime = Arc::new(
10100            Runtime::new(
10101                builder.build().unwrap(),
10102                RuntimeConfig {
10103                    policy_mode: PolicyMode::Bypass,
10104                    agent_swarm_mode: false,
10105                    ultra_mode: false,
10106                    ..RuntimeConfig::default()
10107                },
10108            )
10109            .unwrap(),
10110        );
10111
10112        let result = runtime
10113            .route_tool_call(
10114                &"thread-deadline-task".to_string(),
10115                &"turn-deadline-task".to_string(),
10116                ToolCallCompleted {
10117                    id: "task-1".to_string(),
10118                    name: "task".to_string(),
10119                    arguments: serde_json::json!({
10120                        "description": "inspect",
10121                        "prompt": "read"
10122                    })
10123                    .to_string(),
10124                },
10125                None,
10126                Some(OffsetDateTime::now_utc() + Duration::seconds(1)),
10127            )
10128            .await
10129            .unwrap();
10130
10131        assert!(result.is_error);
10132        assert!(result.result.contains("deadline policy skipped"));
10133        assert_eq!(*calls.lock().unwrap(), 0);
10134    }
10135
10136    struct TestToolContributor {
10137        tool: Arc<dyn ToolExecutor>,
10138    }
10139
10140    impl ToolContributor for TestToolContributor {
10141        fn id(&self) -> String {
10142            "test-tool".to_string()
10143        }
10144
10145        fn contribute(&self, registry: &mut ToolRegistry) -> anyhow::Result<()> {
10146            registry.register(self.tool.clone())
10147        }
10148    }
10149
10150    #[tokio::test]
10151    async fn agent_swarm_mode_override_is_per_thread() {
10152        let runtime = Runtime::fake().unwrap();
10153        let trigger = roder_api::subagents::AgentSwarmModeTrigger::Manual;
10154
10155        // Off everywhere by default.
10156        assert!(
10157            !runtime
10158                .effective_agent_swarm_mode_for_thread("thread-a")
10159                .await
10160        );
10161        assert!(
10162            !runtime
10163                .effective_agent_swarm_mode_for_thread("thread-b")
10164                .await
10165        );
10166
10167        // Enabling on thread-a does not leak into thread-b.
10168        assert!(
10169            runtime
10170                .set_agent_swarm_mode_for_thread("thread-a", true, trigger)
10171                .await
10172        );
10173        assert!(
10174            runtime
10175                .effective_agent_swarm_mode_for_thread("thread-a")
10176                .await
10177        );
10178        assert!(
10179            !runtime
10180                .effective_agent_swarm_mode_for_thread("thread-b")
10181                .await
10182        );
10183
10184        // A per-thread `off` override wins over a runtime-global `on` default.
10185        runtime.set_agent_swarm_mode(true, trigger).await.unwrap();
10186        runtime
10187            .set_agent_swarm_mode_for_thread("thread-b", false, trigger)
10188            .await;
10189        assert!(
10190            !runtime
10191                .effective_agent_swarm_mode_for_thread("thread-b")
10192                .await,
10193            "explicit per-thread off overrides the global on default"
10194        );
10195        // A thread with no override still follows the global default.
10196        assert!(
10197            runtime
10198                .effective_agent_swarm_mode_for_thread("thread-c")
10199                .await,
10200            "threads without an override follow the runtime-global default"
10201        );
10202    }
10203
10204    #[tokio::test]
10205    async fn set_agent_swarm_mode_for_thread_emits_event_with_real_thread_id() {
10206        let runtime = Runtime::fake().unwrap();
10207        let mut events = runtime.subscribe_events();
10208        runtime
10209            .set_agent_swarm_mode_for_thread(
10210                "thread-xyz",
10211                true,
10212                roder_api::subagents::AgentSwarmModeTrigger::Task,
10213            )
10214            .await;
10215        let mut saw = false;
10216        for _ in 0..8 {
10217            let envelope = tokio::time::timeout(std::time::Duration::from_secs(2), events.recv())
10218                .await
10219                .unwrap()
10220                .unwrap();
10221            if let RoderEvent::AgentSwarmModeChanged(event) = envelope.event {
10222                assert_eq!(event.thread_id, "thread-xyz");
10223                assert!(event.enabled);
10224                assert_eq!(
10225                    event.trigger,
10226                    roder_api::subagents::AgentSwarmModeTrigger::Task
10227                );
10228                saw = true;
10229                break;
10230            }
10231        }
10232        assert!(
10233            saw,
10234            "expected an AgentSwarmModeChanged event for the thread"
10235        );
10236    }
10237
10238    #[tokio::test]
10239    async fn ultra_mode_override_is_per_thread() {
10240        let runtime = Runtime::fake().unwrap();
10241        let trigger = roder_api::subagents::UltraModeTrigger::Manual;
10242
10243        assert!(!runtime.effective_ultra_mode_for_thread("thread-a").await);
10244        assert!(!runtime.effective_ultra_mode_for_thread("thread-b").await);
10245
10246        assert!(
10247            runtime
10248                .set_ultra_mode_for_thread("thread-a", true, trigger)
10249                .await
10250        );
10251        assert!(runtime.effective_ultra_mode_for_thread("thread-a").await);
10252        assert!(!runtime.effective_ultra_mode_for_thread("thread-b").await);
10253
10254        runtime.set_ultra_mode(true, trigger).await.unwrap();
10255        runtime
10256            .set_ultra_mode_for_thread("thread-b", false, trigger)
10257            .await;
10258        assert!(
10259            !runtime.effective_ultra_mode_for_thread("thread-b").await,
10260            "explicit per-thread off overrides the global on default"
10261        );
10262        assert!(
10263            runtime.effective_ultra_mode_for_thread("thread-c").await,
10264            "threads without an override follow the runtime-global default"
10265        );
10266    }
10267
10268    #[tokio::test]
10269    async fn set_ultra_mode_for_thread_emits_event_with_real_thread_id() {
10270        let runtime = Runtime::fake().unwrap();
10271        let mut events = runtime.subscribe_events();
10272        runtime
10273            .set_ultra_mode_for_thread(
10274                "thread-ultra",
10275                true,
10276                roder_api::subagents::UltraModeTrigger::Task,
10277            )
10278            .await;
10279        let mut saw = false;
10280        for _ in 0..8 {
10281            let envelope = tokio::time::timeout(std::time::Duration::from_secs(2), events.recv())
10282                .await
10283                .unwrap()
10284                .unwrap();
10285            if let RoderEvent::UltraModeChanged(event) = envelope.event {
10286                assert_eq!(event.thread_id, "thread-ultra");
10287                assert!(event.enabled);
10288                assert_eq!(event.trigger, roder_api::subagents::UltraModeTrigger::Task);
10289                saw = true;
10290                break;
10291            }
10292        }
10293        assert!(saw, "expected an UltraModeChanged event for the thread");
10294    }
10295}
10296
10297#[cfg(test)]
10298#[path = "runtime/mailbox_test_helpers.rs"]
10299mod mailbox_test_helpers;