Skip to main content

roder_core/
runtime.rs

1#[path = "runtime/sampling.rs"]
2pub(crate) mod sampling;
3use sampling::{
4    SamplingPreempted, SamplingRetry, completed_call_replayed, preemptible, wait_for_steer,
5};
6#[path = "runtime/sampling_output.rs"]
7mod sampling_output;
8use sampling_output::{OutputContext, SamplingOutput};
9mod compaction_template;
10#[path = "runtime/eager_tools.rs"]
11mod eager_tools;
12mod thread_admission;
13use eager_tools::EagerTools;
14#[path = "runtime/patch_progress.rs"]
15mod patch_progress;
16use patch_progress::PatchProgressTracker;
17
18use std::collections::{HashMap, HashSet};
19use std::path::PathBuf;
20use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
21use std::sync::{Arc, Weak};
22
23use anyhow::Context;
24use futures::StreamExt;
25use futures::future::{AbortHandle, Abortable, BoxFuture, try_join_all};
26use roder_api::catalog::{
27    EDIT_TOOL_EDIT, EDIT_TOOL_PATCH, PROVIDER_GEMINI, REASONING_NONE, REASONING_ULTRA,
28    built_in_model_profile, built_in_model_profile_for_provider, lookup_model,
29    model_supports_reasoning_effort,
30};
31use roder_api::context::PolicyGate;
32use roder_api::events::*;
33use roder_api::extension::ExtensionRegistry;
34use roder_api::inference::{
35    AgentInferenceRequest, HostedWebSearchConfig, HostedWebSearchMode, InferenceEngine,
36    InferenceEvent, InferenceTurnContext, InstructionBundle, ModelHarnessProfile,
37    ModelSchemaPolicy, ModelSelection, OutputConfig, ProviderTurnCleanup, ReasoningConfig,
38    RuntimeHints, RuntimeProfile, TokenUsage, ToolCallCompleted, ToolSearchConfig,
39    ToolSearchConfigOverlay, finish_reason_from_stop_reason,
40};
41use roder_api::inference_routing::{InferenceRoutingOutcome, ModelSelectionMode};
42use roder_api::lifecycle::{
43    LifecycleMetricsSnapshot, TurnCleanupOwnership, TurnCleanupState, TurnLifecycleReason,
44    TurnLifecycleRecord, TurnLifecycleSnapshot, TurnLifecycleState, turn_lifecycle_snapshot,
45};
46use roder_api::policy_mode::{PolicyDecision, PolicyMode};
47use roder_api::reliability::{
48    ReliabilityContext, ReliabilityDetails, ReliabilityErrorClass, ReliabilityLimitDecision,
49    ReliabilityLimitRecorded, ReliabilityRequestPolicy, ReliabilityRetryDecision,
50    ReliabilityRetryRecorded, provider_retry_delay_ms,
51};
52use roder_api::remote_runner::{
53    RemoteRunnerProvider, RemoteRunnerSession, RemoteWorkspace, RunnerDestination,
54    RunnerSessionState, ThreadRunnerBinding,
55};
56use roder_api::subagents::SubagentDefinition;
57use roder_api::teams::{
58    TeamId, TeamMailboxMessage, TeamMailboxMessageKind, TeamMemberDescriptor, TeamMemberRole,
59    TeamMemberStatus,
60};
61use roder_api::thread::{
62    ThreadItemEvent, ThreadItemEventKind, ThreadMetadata, ThreadSnapshot, ThreadStore,
63    ThreadUsageMetadata, is_synthetic_event_thread_id, validate_thread_workspace,
64};
65use roder_api::tools::{ToolCall, ToolChoice, ToolExecutionContext, ToolRegistry, ToolResult};
66use roder_api::transcript::{
67    AssistantMessage, ErrorRecord, InputImage, ReasoningSummary, ToolCallRecord, ToolResultRecord,
68    TranscriptItem, UserMessage,
69};
70use roder_sandbox::ScopedFilesystem;
71use roder_sandbox::process::LocalProcessRunner;
72use roder_skills::{SkillRegistry, SkillRegistryOptions};
73use time::{Duration, OffsetDateTime};
74use tokio::sync::{Mutex, Notify, RwLock, oneshot};
75
76mod codex_v2;
77
78use crate::artifacts::{
79    ContextArtifactStore as FilesystemContextArtifactStore, default_context_artifact_dir,
80};
81use crate::bus::EventBus;
82use crate::dynamic_workflows::{
83    DynamicWorkflowEffortProfile, RuntimeDynamicWorkflowConfig, WorkflowTriggerDecision,
84    classify_workflow_trigger, ultracode_reasoning_level_for_model,
85};
86use crate::fake_provider::FakeInferenceEngine;
87use crate::goals::RuntimeGoalController;
88use crate::inference_routing::{
89    InferenceRoutingRequest, RuntimeInferenceRouterConfig, collect_inference_routing_candidates,
90    route_inference_selection, transcript_failure_count_since,
91};
92use crate::instructions::{
93    apply_agent_swarm_mode, apply_codex_multi_agent_mode, apply_model_instruction_overlay,
94    apply_parallel_web_tools, apply_plan_mode, apply_runtime_profile, apply_task_ledger_required,
95    apply_thread_developer_instructions, apply_turn_developer_context,
96};
97use crate::policy_gate::DefaultPolicyGate;
98use crate::reliability::{
99    ReliabilityLimitHit, RuntimeReliabilityConfig, TurnReliabilityState,
100    provider_stream_retry_cause,
101};
102pub use crate::speed_policy::RuntimeSpeedPolicyConfig;
103use crate::speed_policy::{SpeedPolicyState, reasoning_from_decision};
104use crate::subagent_traces::{RuntimeAgentSwarmProgressSink, RuntimeSubagentTraceSink};
105use crate::teams::{TeamManager, TeamMemberStartRequest, TeamStartRequest, TeamState};
106use crate::thread_item_cache::{ThreadItemCache, ThreadItemCacheEntry};
107use crate::verification_gate::VerificationGateState;
108
109const MAX_TOOL_ROUNDS_PER_TURN: usize = 1024;
110/// Injected when `continue_on_failure_limit` turns a consecutive-tool-failure
111/// limit into a continuation, or as the empty-tool-call persistence nudge, so
112/// the model keeps working instead of ending the turn early.
113const RELIABILITY_CONTINUATION_PROMPT: &str = "Verify the task is fully complete. If any part remains unfinished or unverified, keep working using the available tools — try an alternative approach if one is failing. Only stop once the solution is complete and verified; if it is already complete, restate your final answer.";
114/// Capacity of the runtime event broadcast ring buffer. The TUI drains this on
115/// its render loop, which during an active turn only wakes at the ~6 FPS status
116/// animation cadence (backend events do not wake the input poll), so up to
117/// ~166ms of events buffer between drains. A reasoning-high provider that runs
118/// its whole tool loop inside one turn (e.g. claude-code) streams a firehose of
119/// `InferenceEventReceived` deltas plus per-step tool/thinking events; the old
120/// 1024-slot buffer overflowed in those windows and silently dropped tool and
121/// thinking rows from the live view. Sized for generous headroom across bursts
122/// and brief render stalls.
123const EVENT_BUS_CAPACITY: usize = 16_384;
124
125/// Turn id reported to session-scoped local hooks. `SessionStart` and
126/// `SessionEnd` bracket the session rather than any one turn, but the hook
127/// payload carries a turn id, so they share this synthetic value.
128const SESSION_HOOK_TURN_ID: &str = "session";
129pub(crate) const FINAL_ANSWER_PHASE: &str = "final_answer";
130pub(crate) const TASK_LEDGER_TOOL_NAME: &str = "task_ledger.update";
131const TASK_LEDGER_COMPLETION_REMINDER_LIMIT: u8 = 2;
132const TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS: u64 = 180;
133const TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT: u8 = 1;
134pub(crate) const MIN_CHILD_DEADLINE_SECONDS: u64 = 2;
135const MODEL_PROFILE_TRACE_KIND: &str = "model_profile_segment";
136const MODEL_SWITCH_SUMMARY_PREFIX: &str = "Model switch summary:";
137const PROVIDER_CLEANUP_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5);
138
139#[derive(Clone, Copy, Debug, Eq, PartialEq)]
140enum InferenceTimeoutAction {
141    ScoreableCheckpoint,
142    Finalization,
143}
144
145#[derive(Debug, Clone)]
146pub struct RuntimeConfig {
147    pub default_provider: String,
148    pub default_model: String,
149    pub reasoning: Option<String>,
150    pub auto_compact_token_limit: Option<u32>,
151    pub file_backed_dynamic_context: bool,
152    pub hosted_web_search: HostedWebSearchConfig,
153    pub tool_search: ToolSearchConfig,
154    pub provider_tool_search: HashMap<String, ToolSearchConfigOverlay>,
155    pub model_tool_search: HashMap<String, ToolSearchConfigOverlay>,
156    pub model_edit_tools: HashMap<String, String>,
157    pub model_parallel_tool_calls: HashMap<String, bool>,
158    pub model_profiles: HashMap<String, ModelHarnessProfile>,
159    pub tool_allowlist: Vec<String>,
160    /// Seconds a host-executed external tool call may stay unresolved before it fails with a timeout error.
161    pub external_tool_timeout_seconds: u64,
162    pub command_shell: String,
163    pub workspace: Option<String>,
164    pub policy_mode: PolicyMode,
165    /// Whether agent-swarm mode is active (roadmap 104). When on, the runtime
166    /// injects the swarm reminder into each turn's developer instructions so any
167    /// client benefits, not just the TUI.
168    pub agent_swarm_mode: bool,
169    /// Whether ultra mode is active. When on, the runtime injects proactive
170    /// multi-agent delegation policy for any model (not only Codex Sol/Terra
171    /// Ultra effort). Selecting Ultra reasoning on Sol/Terra still enables
172    /// proactive multi-agent for that model without requiring this flag.
173    pub ultra_mode: bool,
174    pub runtime_profile: RuntimeProfile,
175    pub inference_router: RuntimeInferenceRouterConfig,
176    pub speed_policy: RuntimeSpeedPolicyConfig,
177    pub dynamic_workflows: RuntimeDynamicWorkflowConfig,
178    pub reliability: RuntimeReliabilityConfig,
179    /// Positive wall-clock limit for a turn in any runtime profile. `None` or zero is unbounded.
180    pub turn_deadline_seconds: Option<u64>,
181    pub remote_runner_destination: Option<RunnerDestination>,
182    pub team_data_dir: Option<PathBuf>,
183    pub roadmap_data_dir: Option<PathBuf>,
184    pub media_generation: crate::media_generation::RuntimeMediaGenerationConfig,
185    /// `[review]` settings: the review sub-turn and its publishers.
186    pub review: crate::review::RuntimeReviewConfig,
187}
188
189impl Default for RuntimeConfig {
190    fn default() -> Self {
191        Self {
192            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
193            default_model: "mock".to_string(),
194            reasoning: None,
195            auto_compact_token_limit: None,
196            file_backed_dynamic_context: true,
197            hosted_web_search: HostedWebSearchConfig::cached(),
198            tool_search: ToolSearchConfig::default(),
199            provider_tool_search: HashMap::new(),
200            model_tool_search: HashMap::new(),
201            model_edit_tools: HashMap::new(),
202            model_parallel_tool_calls: HashMap::new(),
203            model_profiles: HashMap::new(),
204            tool_allowlist: Vec::new(),
205            external_tool_timeout_seconds: DEFAULT_EXTERNAL_TOOL_TIMEOUT_SECONDS,
206            command_shell: roder_api::command_shell::default_command_shell(),
207            workspace: None,
208            policy_mode: PolicyMode::Default,
209            agent_swarm_mode: false,
210            ultra_mode: false,
211            runtime_profile: RuntimeProfile::Interactive,
212            inference_router: RuntimeInferenceRouterConfig::default(),
213            speed_policy: RuntimeSpeedPolicyConfig::default(),
214            dynamic_workflows: RuntimeDynamicWorkflowConfig::default(),
215            reliability: RuntimeReliabilityConfig::default(),
216            turn_deadline_seconds: None,
217            remote_runner_destination: None,
218            team_data_dir: None,
219            roadmap_data_dir: None,
220            media_generation: crate::media_generation::RuntimeMediaGenerationConfig::default(),
221            review: crate::review::RuntimeReviewConfig::default(),
222        }
223    }
224}
225
226#[derive(Debug, Clone)]
227pub struct StartTurnRequest {
228    pub thread_id: ThreadId,
229    pub message: String,
230    pub images: Vec<InputImage>,
231    pub provider_override: Option<String>,
232    pub model_override: Option<String>,
233    pub reasoning_override: Option<String>,
234    pub workspace: String,
235    pub instructions: InstructionBundle,
236    /**
237     * Per-turn developer-authority context for this turn's InstructionBundle.
238     * Applies to every inference round of the turn, is never written to
239     * thread state, and does not carry over to later turns.
240     */
241    pub developer_context: Option<String>,
242    pub task_ledger_required: bool,
243    /**
244     * Per-turn provider service tier (OpenAI `service_tier`, e.g.
245     * `"priority"` for Fast mode). Carried to every inference round of the
246     * turn as `RuntimeHints::service_tier`; `None` keeps the provider
247     * default. Providers without a tier concept ignore it.
248     */
249    pub service_tier_override: Option<String>,
250}
251
252#[derive(Debug, Clone)]
253pub struct CreateThreadRequest {
254    pub title: Option<String>,
255    pub workspace: String,
256    pub workspace_id: Option<String>,
257    pub root_id: Option<String>,
258    pub provider: Option<String>,
259    pub model: Option<String>,
260    pub selection_mode: Option<ModelSelectionMode>,
261    /// Per-thread tool filter applied on top of the runtime allowlist. Empty = no filtering.
262    pub tool_allowlist: Vec<String>,
263    /// Host-supplied instructions added to the developer slot of every turn's inference request.
264    pub developer_instructions: Option<String>,
265    /// Host-executed tool specs advertised to the model on every turn of this thread.
266    pub external_tools: Vec<roder_api::tools::ToolSpec>,
267    /// Explicit remote-runner binding for the thread's native coding tools.
268    pub runner: Option<ThreadRunnerSelection>,
269}
270
271/**
272 * Thread-level remote-runner selection. The destination config is persisted
273 * with the thread, so secrets must reach the provider through its
274 * environment, not this config.
275 */
276#[derive(Debug, Clone)]
277pub struct ThreadRunnerSelection {
278    pub provider_id: String,
279    pub config: serde_json::Value,
280    /// Absolute path on the runner used as the thread's coding-tool workspace root.
281    pub workspace: String,
282    /**
283     * Extra absolute runner paths file reads may resolve under, beyond
284     * `workspace`. Writes and the working directory stay confined to
285     * `workspace`.
286     */
287    pub read_roots: Vec<String>,
288}
289
290#[derive(Debug, Clone, PartialEq, Eq)]
291pub struct PendingPlanExit {
292    pub thread_id: ThreadId,
293    pub turn_id: TurnId,
294    pub request_id: String,
295    pub target_mode: PolicyMode,
296    pub plan_summary: Option<String>,
297    pub next_steps: Vec<String>,
298    pub requested_at: OffsetDateTime,
299    pub expires_at: Option<OffsetDateTime>,
300}
301
302pub(crate) struct PendingToolApproval {
303    pub(crate) thread_id: ThreadId,
304    pub(crate) turn_id: TurnId,
305    pub(crate) tool_id: String,
306    pub(crate) tool_name: String,
307    pub(crate) call: roder_api::tools::ToolCall,
308    pub(crate) tx: oneshot::Sender<bool>,
309}
310
311pub(crate) struct PendingUserInput {
312    pub(crate) thread_id: ThreadId,
313    pub(crate) turn_id: TurnId,
314    pub(crate) tx: oneshot::Sender<serde_json::Value>,
315}
316
317/// Host answer to an external tool call delivered via `tools/resolve`.
318#[derive(Debug, Clone, PartialEq, Eq)]
319pub struct ExternalToolResolution {
320    pub output: String,
321    pub is_error: bool,
322}
323
324pub(crate) struct PendingExternalToolCall {
325    pub(crate) thread_id: ThreadId,
326    pub(crate) turn_id: TurnId,
327    pub(crate) tool_id: String,
328    pub(crate) tool_name: String,
329    pub(crate) tx: oneshot::Sender<ExternalToolResolution>,
330}
331
332#[derive(Clone)]
333struct ActiveTurnHandle {
334    thread_id: ThreadId,
335    abort: AbortHandle,
336    steers: Arc<Mutex<Vec<QueuedTurnSteer>>>,
337    steer_changed: tokio::sync::watch::Sender<u64>,
338    drain: Arc<TurnDrainHandle>,
339}
340
341/// Tracks a task after it has been interrupted and removed from the interactive
342/// active-turn map. This lets users start a follow-up turn immediately while a
343/// bounded runtime shutdown can still await the original task's cleanup.
344struct TurnDrainHandle {
345    thread_id: ThreadId,
346    interrupt_requested: AtomicBool,
347    interrupt_reason: Mutex<Option<TurnLifecycleReason>>,
348    completed: AtomicBool,
349    completed_notify: Notify,
350}
351
352/// Result of a bounded runtime drain. `Clean` proves the runtime's own turn
353/// tasks have reached their cleanup paths; provider-specific child-process
354/// reaping remains an explicit provider capability.
355#[derive(Debug, Clone, PartialEq, Eq)]
356pub enum RuntimeDrainOutcome {
357    Clean {
358        interrupted_turn_ids: Vec<TurnId>,
359    },
360    DeadlineExceeded {
361        interrupted_turn_ids: Vec<TurnId>,
362        remaining_turn_ids: Vec<TurnId>,
363    },
364    /// At least one lifecycle transition initiated by this drain could not be
365    /// durably appended. The runtime still made its best cleanup attempt, but
366    /// callers must not claim a persisted terminal state as proof of recovery.
367    PersistenceFailed {
368        interrupted_turn_ids: Vec<TurnId>,
369        remaining_turn_ids: Vec<TurnId>,
370    },
371}
372
373#[derive(Clone)]
374struct QueuedTurnSteer {
375    message: UserMessage,
376    mailbox_ack: Option<MailboxDeliveryAck>,
377}
378
379#[derive(Clone)]
380struct MailboxDeliveryAck {
381    team_id: TeamId,
382    message_ids: Vec<String>,
383}
384
385#[derive(Clone)]
386struct InheritedTurnContext {
387    workspace: String,
388    instructions: InstructionBundle,
389    developer_context: Option<String>,
390}
391
392#[derive(Debug, Clone, Default, PartialEq, Eq)]
393pub struct ThreadActivity {
394    pub active_turn_id: Option<TurnId>,
395    pub active_flags: Vec<String>,
396}
397
398/// Per-thread settings persisted at thread creation and applied to every turn.
399#[derive(Debug, Clone, Default)]
400pub(crate) struct ThreadTurnOverrides {
401    pub(crate) tool_allowlist: Vec<String>,
402    pub(crate) developer_instructions: Option<String>,
403    pub(crate) external_tools: Vec<roder_api::tools::ToolSpec>,
404}
405
406#[derive(Debug, Clone, Copy, PartialEq, Eq)]
407pub(crate) enum TurnRunOutcome {
408    Completed,
409    Stopped,
410}
411
412impl PendingPlanExit {
413    pub fn new(
414        thread_id: ThreadId,
415        turn_id: TurnId,
416        request_id: String,
417        target_mode: PolicyMode,
418        plan_summary: Option<String>,
419        next_steps: Vec<String>,
420    ) -> Self {
421        let requested_at = OffsetDateTime::now_utc();
422        Self {
423            thread_id,
424            turn_id,
425            request_id,
426            target_mode,
427            plan_summary,
428            next_steps,
429            requested_at,
430            expires_at: Some(requested_at + default_plan_exit_timeout()),
431        }
432    }
433
434    pub fn is_expired(&self, now: OffsetDateTime) -> bool {
435        self.expires_at.is_some_and(|expires_at| now >= expires_at)
436    }
437}
438
439pub fn default_plan_exit_timeout() -> Duration {
440    Duration::minutes(10)
441}
442
443pub const DEFAULT_EXTERNAL_TOOL_TIMEOUT_SECONDS: u64 = 300;
444
445fn format_mailbox_messages(team: &TeamState, messages: &[TeamMailboxMessage]) -> String {
446    messages
447        .iter()
448        .map(|message| {
449            let recipient = team
450                .members
451                .iter()
452                .find(|member| member.id == message.to_member_id)
453                .map(canonical_team_member_path)
454                .unwrap_or_else(|| format!("/root/{}", message.to_member_id));
455            let sender = message
456                .from_member_id
457                .as_deref()
458                .and_then(|id| team.members.iter().find(|member| member.id == id))
459                .map(canonical_team_member_path)
460                .unwrap_or_else(|| "/root".to_string());
461            let message_type = match message.kind {
462                TeamMailboxMessageKind::Message => "MESSAGE",
463                TeamMailboxMessageKind::NewTask => "NEW_TASK",
464                TeamMailboxMessageKind::FinalAnswer => "FINAL_ANSWER",
465            };
466            format!(
467                "Message Type: {message_type}\nTask name: {recipient}\nSender: {sender}\nPayload:\n{}",
468                message.text
469            )
470        })
471        .collect::<Vec<_>>()
472        .join("\n\n")
473}
474
475fn canonical_team_member_path(member: &TeamMemberDescriptor) -> String {
476    if let Some(agent_path) = member
477        .agent_path
478        .as_deref()
479        .filter(|path| *path == "/root" || path.starts_with("/root/"))
480    {
481        return agent_path.to_string();
482    }
483    if member.role == TeamMemberRole::Lead {
484        return "/root".to_string();
485    }
486    member
487        .task_name
488        .as_deref()
489        .filter(|name| !name.trim().is_empty())
490        .map(|name| format!("/root/{}", name.trim().trim_matches('/')))
491        .unwrap_or_else(|| format!("/root/{}", member.id))
492}
493
494fn is_codex_v2_team(team: &TeamState) -> bool {
495    team.members.iter().any(|member| {
496        member
497            .model
498            .as_deref()
499            .is_some_and(|model| model_supports_reasoning_effort(model, REASONING_ULTRA))
500    })
501}
502
503fn runtime_local_team_view(
504    mut team: TeamState,
505    active_thread_ids: &std::collections::HashSet<ThreadId>,
506) -> TeamState {
507    for member in &mut team.members {
508        if member.status == TeamMemberStatus::Running
509            && !active_thread_ids.contains(&member.thread_id)
510        {
511            member.status = TeamMemberStatus::Interrupted;
512            member.current_turn_id = None;
513        }
514    }
515    team
516}
517
518type RunnerToolExecutionKey = (String, String);
519type RunnerToolExecutionMutex = Mutex<()>;
520type RunnerToolExecutionLockRegistry =
521    Mutex<HashMap<RunnerToolExecutionKey, Weak<RunnerToolExecutionMutex>>>;
522
523pub struct Runtime {
524    pub bus: EventBus,
525    pub registry: ExtensionRegistry,
526    config: RwLock<RuntimeConfig>,
527    pending_plan_exit: RwLock<Option<PendingPlanExit>>,
528    pub(crate) pending_tool_approvals: Mutex<HashMap<String, PendingToolApproval>>,
529    pub(crate) pending_user_inputs: Mutex<HashMap<String, PendingUserInput>>,
530    pub(crate) pending_external_tool_calls: Mutex<HashMap<String, PendingExternalToolCall>>,
531    /// Serializes turn admission with shutdown quiescing. A drain takes this
532    /// gate before flipping `accepting_turns` and snapshotting active work so a
533    /// turn that observed the old flag cannot be inserted after the snapshot.
534    turn_admission: Mutex<()>,
535    active_turns: RwLock<HashMap<TurnId, ActiveTurnHandle>>,
536    turn_drains: RwLock<HashMap<TurnId, Arc<TurnDrainHandle>>>,
537    /// Provider cleanup handles register synchronously through the inference
538    /// tool-executor context. They are intentionally separate from active turn
539    /// admission so an interrupted turn can remain drainable after a follow-up
540    /// turn starts on the same thread.
541    provider_turn_cleanups: std::sync::Mutex<HashMap<TurnId, Arc<dyn ProviderTurnCleanup>>>,
542    accepting_turns: AtomicBool,
543    /// Monotonic counter used by bounded drains to surface lifecycle-state
544    /// persistence failures without making an individual provider turn fail.
545    lifecycle_persistence_failures: AtomicUsize,
546    lifecycle_shutdown_drains: AtomicUsize,
547    lifecycle_clean_shutdowns: AtomicUsize,
548    lifecycle_deadline_exceeded: AtomicUsize,
549    lifecycle_persistence_failed_drains: AtomicUsize,
550    lifecycle_restart_reconciliations: AtomicUsize,
551    lifecycle_shutdown_drain_duration_ms_total: AtomicUsize,
552    provider_cleanup_confirmed: AtomicUsize,
553    provider_cleanup_timed_out: AtomicUsize,
554    provider_cleanup_unknown: AtomicUsize,
555    active_turns_changed: Notify,
556    active_turn_selections: RwLock<HashMap<TurnId, ModelSelectionMode>>,
557    compaction_templates: RwLock<HashMap<ThreadId, AgentInferenceRequest>>,
558    thread_admission_gates: Mutex<HashMap<ThreadId, Arc<Mutex<()>>>>,
559    /// Threads that have already dispatched a `SessionStart` local hook.
560    ///
561    /// `ThreadCreated` fires once, but `ThreadLoaded` fires on every read of the
562    /// thread from its store, so without this a session ran its setup hooks
563    /// several times — including after the turn had already stopped.
564    session_hook_started: RwLock<HashSet<ThreadId>>,
565    active_turn_contexts: RwLock<HashMap<TurnId, InheritedTurnContext>>,
566    /// Process-local execution boundary. Local/CLI runtimes default to true;
567    /// hosted runtime pools set this false so missing runner metadata fails
568    /// closed instead of exposing the host workspace.
569    allow_local_workspaces: AtomicBool,
570    // Spawn-time live authority retained for every reusable turn of a long-lived teammate.
571    // This stays process-local because developer_context is explicitly volatile and must never
572    // be persisted to thread state.
573    team_member_turn_contexts: Mutex<HashMap<ThreadId, InheritedTurnContext>>,
574    workspace: PathBuf,
575    pub(crate) teams: TeamManager,
576    agent_team_spawn_lock: Mutex<()>,
577    pub(crate) roadmaps: Mutex<roder_roadmap::RoadmapRuntime>,
578    /// Completed reviews, most recent last, so a later publish can resolve a
579    /// review id back to its findings. Bounded; see `review::run`.
580    pub(crate) reviews: Mutex<Vec<crate::review::ReviewRecord>>,
581    pub(crate) goals: Arc<RuntimeGoalController>,
582    context_artifacts: roder_api::artifacts::ContextArtifactStore,
583    pub(crate) thread_store: Option<Arc<dyn ThreadStore>>,
584    thread_item_cache: Mutex<ThreadItemCache>,
585    pub(crate) tool_registry: ToolRegistry,
586    media_generation: Arc<crate::media_generation::MediaGenerationService>,
587    pub(crate) skills: RwLock<SkillRegistry>,
588    /// Lazily-started bounded dispatch of emitted events to registry
589    /// `EventSink`s (process extensions etc.); see `event_sink_dispatch`.
590    event_sink_dispatcher: tokio::sync::OnceCell<crate::event_sink_dispatch::EventSinkDispatcher>,
591    pub(crate) compaction_hysteresis:
592        std::sync::Mutex<HashMap<ThreadId, crate::compaction_runtime::CompactionState>>,
593    /// Per-thread agent-swarm mode overrides (roadmap 104). A thread present in
594    /// this map uses its stored value; absent threads fall back to the
595    /// runtime-global `RuntimeConfig.agent_swarm_mode` default. This mirrors the
596    /// team per-member policy-mode override idiom so swarm mode is per-thread
597    /// like the other `thread/*` operations, without a separate runtime.
598    agent_swarm_modes: RwLock<HashMap<ThreadId, bool>>,
599    /// Per-thread ultra mode overrides. A thread present in this map uses its
600    /// stored value; absent threads fall back to the runtime-global
601    /// `RuntimeConfig.ultra_mode` default.
602    ultra_modes: RwLock<HashMap<ThreadId, bool>>,
603    /**
604     * Live remote-runner sessions keyed by thread. Each per-thread mutex is
605     * held across provider initialization, making the first workspace-tool
606     * access a singleflight while allowing unrelated threads to initialize in
607     * parallel. Failed initialization is shared with current waiters, then
608     * evicted so a later tool call can retry.
609     */
610    runner_sessions: Arc<Mutex<HashMap<ThreadId, Arc<RunnerSessionSlot>>>>,
611    /**
612     * Outer tool-call locks keyed by the actual remote session, not the
613     * per-thread destination id. Hosted runtimes are tenant-scoped, so this
614     * serializes threads that share one sandbox without coupling different
615     * tenants or independent runner sessions.
616     */
617    runner_tool_execution_locks: RunnerToolExecutionLockRegistry,
618}
619
620#[derive(Clone)]
621struct CachedRunnerSession {
622    destination: RunnerDestination,
623    session: Arc<dyn RemoteRunnerSession>,
624}
625
626struct RunnerSessionSlot {
627    provider_id: String,
628    destination_id: String,
629    state: Mutex<RunnerSessionSlotState>,
630    initialized: Notify,
631}
632
633enum RunnerSessionSlotState {
634    Empty,
635    Initializing,
636    Ready(CachedRunnerSession),
637    Failed(Arc<str>),
638}
639
640impl RunnerSessionSlot {
641    fn empty(destination: &RunnerDestination) -> Self {
642        Self {
643            provider_id: destination.provider_id.clone(),
644            destination_id: destination.id.clone(),
645            state: Mutex::new(RunnerSessionSlotState::Empty),
646            initialized: Notify::new(),
647        }
648    }
649
650    fn ready(session: CachedRunnerSession) -> Self {
651        Self {
652            provider_id: session.destination.provider_id.clone(),
653            destination_id: session.destination.id.clone(),
654            state: Mutex::new(RunnerSessionSlotState::Ready(session)),
655            initialized: Notify::new(),
656        }
657    }
658
659    fn matches(&self, destination: &RunnerDestination) -> bool {
660        self.provider_id == destination.provider_id && self.destination_id == destination.id
661    }
662}
663
664impl Runtime {
665    pub fn new(registry: ExtensionRegistry, config: RuntimeConfig) -> anyhow::Result<Self> {
666        if registry.inference_engines.is_empty() {
667            anyhow::bail!("at least one inference engine must be registered");
668        }
669        validate_runtime_config_reasoning(&config)?;
670        validate_runtime_inference_router_config(&registry, &config)?;
671
672        let bus = EventBus::new(EVENT_BUS_CAPACITY);
673        let thread_store = registry
674            .thread_stores
675            .first()
676            .map(|factory| factory.create());
677        let mut tool_registry = ToolRegistry::default();
678        for contributor in &registry.tools {
679            contributor
680                .contribute(&mut tool_registry)
681                .with_context(|| format!("tool contributor {} failed", contributor.id()))?;
682        }
683        crate::agent_control_tools::contribute_agent_control_tools(&mut tool_registry)?;
684
685        let media_generation = Arc::new(crate::media_generation::MediaGenerationService::new(
686            registry.media_generator_providers.clone(),
687            config.media_generation.clone(),
688        ));
689        tool_registry.replace(Arc::new(
690            crate::media_generation::MediaGenerateImageTool::new(media_generation.clone()),
691        ));
692
693        let team_data_dir = config.team_data_dir.clone();
694        let workspace = config
695            .workspace
696            .clone()
697            .map(PathBuf::from)
698            .unwrap_or(std::env::current_dir()?);
699        let roadmap_data_dir = config
700            .roadmap_data_dir
701            .clone()
702            .unwrap_or_else(|| workspace.join(".roder"));
703        let context_artifacts = thread_store
704            .as_ref()
705            .and_then(|store| store.context_artifact_store())
706            .or_else(|| {
707                thread_store
708                    .as_ref()
709                    .and_then(|store| store.local_thread_root())
710                    .map(FilesystemContextArtifactStore::shared_thread_scoped)
711            })
712            .unwrap_or_else(|| {
713                FilesystemContextArtifactStore::shared_legacy(default_context_artifact_dir())
714            });
715        let goals = Arc::new(RuntimeGoalController::new(
716            bus.clone(),
717            thread_store.clone(),
718        ));
719        let runtime = Self {
720            bus,
721            registry,
722            config: RwLock::new(config),
723            pending_plan_exit: RwLock::new(None),
724            pending_tool_approvals: Mutex::new(HashMap::new()),
725            pending_user_inputs: Mutex::new(HashMap::new()),
726            pending_external_tool_calls: Mutex::new(HashMap::new()),
727            turn_admission: Mutex::new(()),
728            active_turns: RwLock::new(HashMap::new()),
729            turn_drains: RwLock::new(HashMap::new()),
730            provider_turn_cleanups: std::sync::Mutex::new(HashMap::new()),
731            accepting_turns: AtomicBool::new(true),
732            lifecycle_persistence_failures: AtomicUsize::new(0),
733            lifecycle_shutdown_drains: AtomicUsize::new(0),
734            lifecycle_clean_shutdowns: AtomicUsize::new(0),
735            lifecycle_deadline_exceeded: AtomicUsize::new(0),
736            lifecycle_persistence_failed_drains: AtomicUsize::new(0),
737            lifecycle_restart_reconciliations: AtomicUsize::new(0),
738            lifecycle_shutdown_drain_duration_ms_total: AtomicUsize::new(0),
739            provider_cleanup_confirmed: AtomicUsize::new(0),
740            provider_cleanup_timed_out: AtomicUsize::new(0),
741            provider_cleanup_unknown: AtomicUsize::new(0),
742            active_turns_changed: Notify::new(),
743            active_turn_selections: RwLock::new(HashMap::new()),
744            compaction_templates: RwLock::new(HashMap::new()),
745            thread_admission_gates: Mutex::new(HashMap::new()),
746            session_hook_started: RwLock::new(HashSet::new()),
747            active_turn_contexts: RwLock::new(HashMap::new()),
748            allow_local_workspaces: AtomicBool::new(true),
749            team_member_turn_contexts: Mutex::new(HashMap::new()),
750            workspace: workspace.clone(),
751            teams: TeamManager::new(
752                team_data_dir.unwrap_or_else(crate::teams::default_team_data_dir),
753            ),
754            agent_team_spawn_lock: Mutex::new(()),
755            roadmaps: Mutex::new(roder_roadmap::RoadmapRuntime::new(
756                workspace,
757                roadmap_data_dir,
758            )),
759            reviews: Mutex::new(Vec::new()),
760            goals,
761            context_artifacts,
762            thread_store,
763            thread_item_cache: Mutex::new(ThreadItemCache::default()),
764            tool_registry,
765            media_generation,
766            skills: RwLock::new(SkillRegistry::load(SkillRegistryOptions::new(
767                PathBuf::new(),
768            ))),
769            event_sink_dispatcher: tokio::sync::OnceCell::new(),
770            compaction_hysteresis: crate::compaction_runtime::compaction_hysteresis_state(),
771            agent_swarm_modes: RwLock::new(HashMap::new()),
772            ultra_modes: RwLock::new(HashMap::new()),
773            runner_sessions: Arc::new(Mutex::new(HashMap::new())),
774            runner_tool_execution_locks: Mutex::new(HashMap::new()),
775        };
776        runtime.bus.emit(RoderEvent::RuntimeStarted(RuntimeStarted {
777            timestamp: OffsetDateTime::now_utc(),
778        }));
779        for manifest in &runtime.registry.manifests {
780            runtime
781                .bus
782                .emit(RoderEvent::ExtensionRegistered(ExtensionRegistered {
783                    extension_id: manifest.id.clone(),
784                    timestamp: OffsetDateTime::now_utc(),
785                }));
786        }
787        Ok(runtime)
788    }
789
790    pub fn from_engine(engine: Arc<dyn InferenceEngine>) -> anyhow::Result<Self> {
791        let mut builder = roder_api::extension::ExtensionRegistryBuilder::new();
792        builder.inference_engine(engine);
793        Self::new(builder.build()?, RuntimeConfig::default())
794    }
795
796    pub fn fake() -> anyhow::Result<Self> {
797        Self::from_engine(Arc::new(FakeInferenceEngine))
798    }
799
800    pub fn subscribe_events(&self) -> tokio::sync::broadcast::Receiver<EventEnvelope> {
801        self.bus.subscribe()
802    }
803
804    /// Returns process-local, redacted lifecycle health counters. The snapshot
805    /// intentionally has fixed fields and contains no provider, command,
806    /// process, thread, or turn identifiers.
807    pub fn lifecycle_metrics(&self) -> LifecycleMetricsSnapshot {
808        LifecycleMetricsSnapshot {
809            shutdown_drain_count: self.lifecycle_shutdown_drains.load(Ordering::Acquire) as u64,
810            clean_shutdown_count: self.lifecycle_clean_shutdowns.load(Ordering::Acquire) as u64,
811            deadline_exceeded_count: self.lifecycle_deadline_exceeded.load(Ordering::Acquire)
812                as u64,
813            persistence_failed_count: self
814                .lifecycle_persistence_failed_drains
815                .load(Ordering::Acquire) as u64,
816            restart_reconciliation_count: self
817                .lifecycle_restart_reconciliations
818                .load(Ordering::Acquire) as u64,
819            lifecycle_persistence_failure_count: self
820                .lifecycle_persistence_failures
821                .load(Ordering::Acquire) as u64,
822            shutdown_drain_duration_ms_total: self
823                .lifecycle_shutdown_drain_duration_ms_total
824                .load(Ordering::Acquire) as u64,
825            provider_cleanup_confirmed_count: self
826                .provider_cleanup_confirmed
827                .load(Ordering::Acquire) as u64,
828            provider_cleanup_timed_out_count: self
829                .provider_cleanup_timed_out
830                .load(Ordering::Acquire) as u64,
831            provider_cleanup_unknown_count: self.provider_cleanup_unknown.load(Ordering::Acquire)
832                as u64,
833        }
834    }
835
836    async fn persist_turn_lifecycle_record(&self, record: &TurnLifecycleRecord) -> bool {
837        let Some(store) = &self.thread_store else {
838            return true;
839        };
840        let state = match record.extension_state() {
841            Ok(state) => state,
842            Err(_) => {
843                self.lifecycle_persistence_failures
844                    .fetch_add(1, Ordering::AcqRel);
845                return false;
846            }
847        };
848        // Lifecycle diagnostics must not turn a provider result into a failed
849        // turn merely because a third-party store lacks extension-state support.
850        // JSONL/Postgres stores persist this append-only record atomically at
851        // their own storage boundary.
852        if store
853            .append_extension_state(&record.thread_id, &state)
854            .await
855            .is_err()
856        {
857            self.lifecycle_persistence_failures
858                .fetch_add(1, Ordering::AcqRel);
859            return false;
860        }
861        true
862    }
863
864    async fn record_turn_lifecycle(
865        &self,
866        thread_id: ThreadId,
867        turn_id: TurnId,
868        state: TurnLifecycleState,
869        cleanup: TurnCleanupState,
870        reason: Option<TurnLifecycleReason>,
871    ) -> TurnLifecycleRecord {
872        self.record_turn_lifecycle_with_ownership(
873            thread_id,
874            turn_id,
875            state,
876            cleanup,
877            reason,
878            TurnCleanupOwnership::RuntimeTaskOnly,
879        )
880        .await
881    }
882
883    async fn record_turn_lifecycle_with_ownership(
884        &self,
885        thread_id: ThreadId,
886        turn_id: TurnId,
887        state: TurnLifecycleState,
888        cleanup: TurnCleanupState,
889        reason: Option<TurnLifecycleReason>,
890        ownership: TurnCleanupOwnership,
891    ) -> TurnLifecycleRecord {
892        let record = TurnLifecycleRecord::new(
893            thread_id,
894            turn_id,
895            state,
896            cleanup,
897            reason,
898            OffsetDateTime::now_utc(),
899        )
900        .with_ownership(ownership);
901        let _ = self.persist_turn_lifecycle_record(&record).await;
902        self.emit(RoderEvent::TurnLifecycleUpdated(record.clone()))
903            .await;
904        record
905    }
906
907    pub(crate) fn register_provider_turn_cleanup(
908        &self,
909        turn_id: &TurnId,
910        cleanup: Arc<dyn ProviderTurnCleanup>,
911    ) {
912        if let Ok(mut cleanups) = self.provider_turn_cleanups.lock() {
913            cleanups.insert(turn_id.clone(), cleanup);
914        }
915    }
916
917    fn provider_cleanup_ownership(&self, turn_id: &TurnId) -> TurnCleanupOwnership {
918        self.provider_turn_cleanups
919            .lock()
920            .ok()
921            .and_then(|cleanups| cleanups.get(turn_id).cloned())
922            .map(|cleanup| cleanup.ownership())
923            .unwrap_or(TurnCleanupOwnership::RuntimeTaskOnly)
924    }
925
926    async fn await_provider_turn_cleanup(
927        &self,
928        turn_id: &TurnId,
929    ) -> (TurnCleanupState, TurnCleanupOwnership) {
930        let cleanup = self
931            .provider_turn_cleanups
932            .lock()
933            .ok()
934            .and_then(|mut cleanups| cleanups.remove(turn_id));
935        let Some(cleanup) = cleanup else {
936            return (
937                TurnCleanupState::Unknown,
938                TurnCleanupOwnership::RuntimeTaskOnly,
939            );
940        };
941        let ownership = cleanup.ownership();
942        match tokio::time::timeout(PROVIDER_CLEANUP_TIMEOUT, cleanup.wait_for_cleanup()).await {
943            Ok(Ok(())) => {
944                self.provider_cleanup_confirmed
945                    .fetch_add(1, Ordering::AcqRel);
946                (
947                    TurnCleanupState::Completed,
948                    TurnCleanupOwnership::ProviderCleanupConfirmed,
949                )
950            }
951            Ok(Err(_)) => {
952                self.provider_cleanup_unknown.fetch_add(1, Ordering::AcqRel);
953                (TurnCleanupState::Unknown, ownership)
954            }
955            Err(_) => {
956                self.provider_cleanup_timed_out
957                    .fetch_add(1, Ordering::AcqRel);
958                (TurnCleanupState::TimedOut, ownership)
959            }
960        }
961    }
962
963    fn record_lifecycle_drain_outcome(
964        &self,
965        started_at: tokio::time::Instant,
966        outcome: &RuntimeDrainOutcome,
967    ) {
968        self.lifecycle_shutdown_drains
969            .fetch_add(1, Ordering::AcqRel);
970        self.lifecycle_shutdown_drain_duration_ms_total.fetch_add(
971            started_at.elapsed().as_millis().min(usize::MAX as u128) as usize,
972            Ordering::AcqRel,
973        );
974        match outcome {
975            RuntimeDrainOutcome::Clean { .. } => {
976                self.lifecycle_clean_shutdowns
977                    .fetch_add(1, Ordering::AcqRel);
978            }
979            RuntimeDrainOutcome::DeadlineExceeded { .. } => {
980                self.lifecycle_deadline_exceeded
981                    .fetch_add(1, Ordering::AcqRel);
982            }
983            RuntimeDrainOutcome::PersistenceFailed { .. } => {
984                self.lifecycle_persistence_failed_drains
985                    .fetch_add(1, Ordering::AcqRel);
986            }
987        }
988    }
989
990    fn lifecycle_record_for_event(event: &RoderEvent) -> Option<TurnLifecycleRecord> {
991        match event {
992            RoderEvent::TurnStarted(event) => Some(TurnLifecycleRecord::new(
993                event.thread_id.clone(),
994                event.turn_id.clone(),
995                TurnLifecycleState::Running,
996                TurnCleanupState::NotRequested,
997                None,
998                event.timestamp,
999            )),
1000            // `run_turn` persists completed turns after it has awaited a
1001            // registered provider cleanup handle. Do not let the generic event
1002            // projection overwrite that acknowledgement with an unproven state.
1003            RoderEvent::TurnCompleted(_) => None,
1004            // Some failure paths already have an event before the turn wrapper
1005            // reaches its cleanup acknowledgement. Preserve a durable fallback
1006            // reason here; the wrapper appends a later record with the more
1007            // precise cleanup outcome when a provider registered one.
1008            RoderEvent::TurnFailed(event) => Some(TurnLifecycleRecord::new(
1009                event.thread_id.clone(),
1010                event.turn_id.clone(),
1011                TurnLifecycleState::Failed,
1012                TurnCleanupState::Unknown,
1013                Some(if event.error_kind.as_deref() == Some("deadline_timeout") {
1014                    TurnLifecycleReason::DeadlineExceeded
1015                } else {
1016                    TurnLifecycleReason::ProviderFailure
1017                }),
1018                event.timestamp,
1019            )),
1020            RoderEvent::TurnDeadlineExceeded(_) | RoderEvent::TurnInterrupted(_) => None,
1021            RoderEvent::TurnLifecycleUpdated(_) => None,
1022            _ => None,
1023        }
1024    }
1025
1026    pub fn registry(&self) -> &ExtensionRegistry {
1027        &self.registry
1028    }
1029
1030    pub fn media_generation(&self) -> Arc<crate::media_generation::MediaGenerationService> {
1031        self.media_generation.clone()
1032    }
1033
1034    pub fn context_artifacts(&self) -> roder_api::artifacts::ContextArtifactStore {
1035        self.context_artifacts.clone()
1036    }
1037
1038    pub async fn execute_workflow_tool(
1039        &self,
1040        thread_id: ThreadId,
1041        tool_name: &str,
1042        arguments: serde_json::Value,
1043    ) -> anyhow::Result<ToolResult> {
1044        let Some(executor) = self.tool_registry.get(tool_name) else {
1045            anyhow::bail!("tool not found: {tool_name}");
1046        };
1047        let tool_call = ToolCall {
1048            id: format!("slash-{tool_name}"),
1049            name: tool_name.to_string(),
1050            raw_arguments: serde_json::to_string(&arguments)?,
1051            arguments,
1052            thread_id: thread_id.clone(),
1053            turn_id: "slash-command".to_string(),
1054        };
1055        let runtime_config = self.status().await;
1056        let ctx = self.tool_execution_context(
1057            thread_id,
1058            "slash-command".to_string(),
1059            runtime_config.policy_mode,
1060            runtime_config.workspace.as_deref(),
1061            Some(&runtime_config.command_shell),
1062        );
1063        executor.execute(ctx, tool_call).await
1064    }
1065
1066    pub(crate) fn tool_execution_context(
1067        &self,
1068        thread_id: ThreadId,
1069        turn_id: TurnId,
1070        mode: PolicyMode,
1071        workspace: Option<&str>,
1072        command_shell: Option<&str>,
1073    ) -> ToolExecutionContext {
1074        let mut ctx = ToolExecutionContext::new(thread_id, turn_id, mode)
1075            .with_command_shell(command_shell.unwrap_or_default())
1076            .with_process_runner(Arc::new(LocalProcessRunner))
1077            .with_context_artifacts(self.context_artifacts.backend())
1078            .with_goal_controller(self.goals.clone())
1079            .with_subagent_trace_sink(Arc::new(RuntimeSubagentTraceSink::new(
1080                self.bus.clone(),
1081                self.thread_store.clone(),
1082            )))
1083            .with_swarm_progress_sink(Arc::new(RuntimeAgentSwarmProgressSink::new(
1084                self.bus.clone(),
1085                self.thread_store.clone(),
1086            )));
1087        if let Some(workspace) = workspace {
1088            ctx = ctx.with_workspace_handle(Arc::new(ScopedFilesystem::new(workspace)));
1089        }
1090        ctx
1091    }
1092
1093    pub async fn status(&self) -> RuntimeConfig {
1094        self.config.read().await.clone()
1095    }
1096
1097    pub async fn set_skills(&self, skills: SkillRegistry) {
1098        *self.skills.write().await = skills;
1099    }
1100
1101    pub async fn skills_snapshot(&self) -> SkillRegistry {
1102        self.skills.read().await.clone()
1103    }
1104
1105    pub fn workspace(&self) -> PathBuf {
1106        self.workspace.clone()
1107    }
1108
1109    /// Controls whether native workspace tools may execute without an
1110    /// explicit remote-runner binding. Hosted runtimes disable this after
1111    /// construction; ordinary local runtimes retain the compatible default.
1112    pub fn set_allow_local_workspaces(&self, allowed: bool) {
1113        self.allow_local_workspaces
1114            .store(allowed, Ordering::Relaxed);
1115    }
1116
1117    pub fn allows_local_workspaces(&self) -> bool {
1118        self.allow_local_workspaces.load(Ordering::Relaxed)
1119    }
1120
1121    pub async fn set_remote_runner_destination(&self, destination: Option<RunnerDestination>) {
1122        let lifecycle = destination.as_ref().map(|destination| RunnerLifecycle {
1123            destination_id: destination.id.clone(),
1124            provider_id: destination.provider_id.clone(),
1125            state: "configured".to_string(),
1126            session_id: None,
1127            timestamp: OffsetDateTime::now_utc(),
1128        });
1129        self.config.write().await.remote_runner_destination = destination;
1130        if let Some(lifecycle) = lifecycle {
1131            self.emit(RoderEvent::RunnerLifecycle(lifecycle)).await;
1132        } else {
1133            self.emit(RoderEvent::RunnerLifecycle(RunnerLifecycle {
1134                destination_id: "local".to_string(),
1135                provider_id: "local".to_string(),
1136                state: "local_fallback".to_string(),
1137                session_id: None,
1138                timestamp: OffsetDateTime::now_utc(),
1139            }))
1140            .await;
1141        }
1142    }
1143
1144    pub async fn set_file_backed_dynamic_context(&self, enabled: bool) -> RuntimeConfig {
1145        let mut cfg = self.config.write().await;
1146        cfg.file_backed_dynamic_context = enabled;
1147        cfg.clone()
1148    }
1149
1150    pub async fn set_command_shell(&self, shell: String) -> RuntimeConfig {
1151        let mut cfg = self.config.write().await;
1152        cfg.command_shell = shell;
1153        cfg.clone()
1154    }
1155
1156    pub async fn pending_plan_exit(&self) -> Option<PendingPlanExit> {
1157        let mut pending = self.pending_plan_exit.write().await;
1158        let current = pending.clone()?;
1159        if !current.is_expired(OffsetDateTime::now_utc()) {
1160            return Some(current);
1161        }
1162        *pending = None;
1163        drop(pending);
1164        self.emit_plan_exit_resolved(&current, false, self.status().await.policy_mode)
1165            .await;
1166        None
1167    }
1168
1169    pub async fn set_policy_mode(
1170        &self,
1171        mode: PolicyMode,
1172        reason: Option<String>,
1173    ) -> anyhow::Result<RuntimeConfig> {
1174        let mut cfg = self.config.write().await;
1175        let previous_mode = cfg.policy_mode;
1176        cfg.policy_mode = mode;
1177        let next = cfg.clone();
1178        drop(cfg);
1179        self.emit(RoderEvent::PolicyModeChanged(PolicyModeChanged {
1180            thread_id: "runtime".to_string(),
1181            turn_id: None,
1182            previous_mode,
1183            new_mode: mode,
1184            reason,
1185            timestamp: OffsetDateTime::now_utc(),
1186        }))
1187        .await;
1188        self.auto_resolve_pending_tool_approvals_for_mode(mode)
1189            .await;
1190        Ok(next)
1191    }
1192
1193    /// Toggle agent-swarm mode for the runtime (roadmap 104). When enabled, the
1194    /// swarm reminder is injected into each turn's developer instructions so
1195    /// every app-server/SDK client gets it, not only the TUI.
1196    pub async fn set_agent_swarm_mode(
1197        &self,
1198        enabled: bool,
1199        trigger: roder_api::subagents::AgentSwarmModeTrigger,
1200    ) -> anyhow::Result<RuntimeConfig> {
1201        let mut cfg = self.config.write().await;
1202        cfg.agent_swarm_mode = enabled;
1203        let next = cfg.clone();
1204        drop(cfg);
1205        self.emit(RoderEvent::AgentSwarmModeChanged(
1206            roder_api::subagents::AgentSwarmModeChanged {
1207                thread_id: "runtime".to_string(),
1208                turn_id: None,
1209                enabled,
1210                trigger,
1211                timestamp: OffsetDateTime::now_utc(),
1212            },
1213        ))
1214        .await;
1215        Ok(next)
1216    }
1217
1218    /// Toggle agent-swarm mode for a single thread (roadmap 104). The override is
1219    /// stored per thread so toggling swarm mode on one thread does not leak the
1220    /// reminder into other threads sharing the runtime; absent threads fall back
1221    /// to the runtime-global default. The emitted `AgentSwarmModeChanged` event
1222    /// carries the real `thread_id`.
1223    pub async fn set_agent_swarm_mode_for_thread(
1224        &self,
1225        thread_id: &str,
1226        enabled: bool,
1227        trigger: roder_api::subagents::AgentSwarmModeTrigger,
1228    ) -> bool {
1229        {
1230            let mut modes = self.agent_swarm_modes.write().await;
1231            modes.insert(thread_id.to_string(), enabled);
1232        }
1233        self.emit(RoderEvent::AgentSwarmModeChanged(
1234            roder_api::subagents::AgentSwarmModeChanged {
1235                thread_id: thread_id.to_string(),
1236                turn_id: None,
1237                enabled,
1238                trigger,
1239                timestamp: OffsetDateTime::now_utc(),
1240            },
1241        ))
1242        .await;
1243        enabled
1244    }
1245
1246    /// Resolve whether agent-swarm mode is active for `thread_id`: the per-thread
1247    /// override when present, otherwise the runtime-global default. The turn loop
1248    /// uses this to decide whether to inject the swarm reminder.
1249    pub async fn effective_agent_swarm_mode_for_thread(&self, thread_id: &str) -> bool {
1250        if let Some(enabled) = self.agent_swarm_modes.read().await.get(thread_id).copied() {
1251            return enabled;
1252        }
1253        self.status().await.agent_swarm_mode
1254    }
1255
1256    /// Toggle ultra mode for the runtime. When enabled, proactive multi-agent
1257    /// policy is injected into each turn's developer instructions for any
1258    /// model, so every app-server/SDK client gets it (not only the TUI).
1259    pub async fn set_ultra_mode(
1260        &self,
1261        enabled: bool,
1262        trigger: roder_api::subagents::UltraModeTrigger,
1263    ) -> anyhow::Result<RuntimeConfig> {
1264        let mut cfg = self.config.write().await;
1265        cfg.ultra_mode = enabled;
1266        let next = cfg.clone();
1267        drop(cfg);
1268        self.emit(RoderEvent::UltraModeChanged(
1269            roder_api::subagents::UltraModeChanged {
1270                thread_id: "runtime".to_string(),
1271                turn_id: None,
1272                enabled,
1273                trigger,
1274                timestamp: OffsetDateTime::now_utc(),
1275            },
1276        ))
1277        .await;
1278        Ok(next)
1279    }
1280
1281    /// Toggle ultra mode for a single thread. The override is stored per thread
1282    /// so toggling on one thread does not leak proactive multi-agent policy into
1283    /// other threads sharing the runtime; absent threads fall back to the
1284    /// runtime-global default.
1285    pub async fn set_ultra_mode_for_thread(
1286        &self,
1287        thread_id: &str,
1288        enabled: bool,
1289        trigger: roder_api::subagents::UltraModeTrigger,
1290    ) -> bool {
1291        {
1292            let mut modes = self.ultra_modes.write().await;
1293            modes.insert(thread_id.to_string(), enabled);
1294        }
1295        self.emit(RoderEvent::UltraModeChanged(
1296            roder_api::subagents::UltraModeChanged {
1297                thread_id: thread_id.to_string(),
1298                turn_id: None,
1299                enabled,
1300                trigger,
1301                timestamp: OffsetDateTime::now_utc(),
1302            },
1303        ))
1304        .await;
1305        enabled
1306    }
1307
1308    /// Resolve whether ultra mode is active for `thread_id`: the per-thread
1309    /// override when present, otherwise the runtime-global default. The turn
1310    /// loop uses this (together with Sol/Terra Ultra effort) to decide whether
1311    /// to inject proactive multi-agent policy.
1312    pub async fn effective_ultra_mode_for_thread(&self, thread_id: &str) -> bool {
1313        if let Some(enabled) = self.ultra_modes.read().await.get(thread_id).copied() {
1314            return enabled;
1315        }
1316        self.status().await.ultra_mode
1317    }
1318
1319    pub async fn set_hosted_web_search(
1320        &self,
1321        mode: HostedWebSearchMode,
1322    ) -> anyhow::Result<RuntimeConfig> {
1323        let mut cfg = self.config.write().await;
1324        cfg.hosted_web_search = HostedWebSearchConfig { mode };
1325        Ok(cfg.clone())
1326    }
1327
1328    async fn auto_resolve_pending_tool_approvals_for_mode(&self, mode: PolicyMode) {
1329        let gate = DefaultPolicyGate::new();
1330        let mut pending = self.pending_tool_approvals.lock().await;
1331        let approval_ids = pending
1332            .iter()
1333            .filter_map(|(approval_id, approval)| {
1334                let ctx = ToolExecutionContext::new(
1335                    approval.thread_id.clone(),
1336                    approval.turn_id.clone(),
1337                    mode,
1338                );
1339                matches!(
1340                    gate.decide(&approval.call, mode, &ctx),
1341                    PolicyDecision::AutoApproved { .. }
1342                )
1343                .then_some(approval_id.clone())
1344            })
1345            .collect::<Vec<_>>();
1346        let approvals = approval_ids
1347            .into_iter()
1348            .filter_map(|approval_id| {
1349                pending
1350                    .remove(&approval_id)
1351                    .map(|approval| (approval_id, approval))
1352            })
1353            .collect::<Vec<_>>();
1354        drop(pending);
1355
1356        for (approval_id, approval) in approvals {
1357            let ctx = ToolExecutionContext::new(
1358                approval.thread_id.clone(),
1359                approval.turn_id.clone(),
1360                mode,
1361            );
1362            let decision = gate.decide(&approval.call, mode, &ctx);
1363            self.emit(RoderEvent::PolicyDecisionRecorded(PolicyDecisionRecorded {
1364                thread_id: approval.thread_id.clone(),
1365                turn_id: approval.turn_id.clone(),
1366                tool_id: approval.tool_id.clone(),
1367                tool_name: approval.tool_name.clone(),
1368                mode,
1369                decision,
1370                timestamp: OffsetDateTime::now_utc(),
1371            }))
1372            .await;
1373            if mode == PolicyMode::Bypass {
1374                self.emit(RoderEvent::PolicyBypassActive(PolicyBypassActive {
1375                    thread_id: approval.thread_id.clone(),
1376                    turn_id: approval.turn_id.clone(),
1377                    tool_id: approval.tool_id.clone(),
1378                    tool_name: approval.tool_name.clone(),
1379                    timestamp: OffsetDateTime::now_utc(),
1380                }))
1381                .await;
1382            }
1383            self.emit(RoderEvent::ApprovalResolved(ApprovalResolved {
1384                thread_id: approval.thread_id,
1385                turn_id: approval.turn_id,
1386                approval_id,
1387                tool_id: approval.tool_id,
1388                tool_name: approval.tool_name,
1389                approved: true,
1390                timestamp: OffsetDateTime::now_utc(),
1391            }))
1392            .await;
1393            let _ = approval.tx.send(true);
1394        }
1395    }
1396
1397    pub async fn record_pending_plan_exit(&self, pending: PendingPlanExit) {
1398        *self.pending_plan_exit.write().await = Some(pending.clone());
1399        self.emit(RoderEvent::PolicyExitPlanRequested(
1400            PolicyExitPlanRequested {
1401                thread_id: pending.thread_id,
1402                turn_id: pending.turn_id,
1403                request_id: pending.request_id,
1404                target_mode: pending.target_mode,
1405                plan_summary: pending.plan_summary,
1406                next_steps: pending.next_steps,
1407                timestamp: OffsetDateTime::now_utc(),
1408            },
1409        ))
1410        .await;
1411    }
1412
1413    pub async fn resolve_pending_plan_exit(
1414        &self,
1415        request_id: &str,
1416        approved: bool,
1417    ) -> anyhow::Result<Option<PendingPlanExit>> {
1418        let mut pending = self.pending_plan_exit.write().await;
1419        let Some(current) = pending.clone() else {
1420            return Ok(None);
1421        };
1422        if current.request_id != request_id {
1423            anyhow::bail!("pending plan exit request {request_id:?} was not found");
1424        }
1425        *pending = None;
1426        drop(pending);
1427
1428        let approved = approved && !current.is_expired(OffsetDateTime::now_utc());
1429        let resolved_mode = if approved {
1430            let mut cfg = self.config.write().await;
1431            let previous_mode = cfg.policy_mode;
1432            cfg.policy_mode = current.target_mode;
1433            drop(cfg);
1434            self.emit(RoderEvent::PolicyModeChanged(PolicyModeChanged {
1435                thread_id: current.thread_id.clone(),
1436                turn_id: Some(current.turn_id.clone()),
1437                previous_mode,
1438                new_mode: current.target_mode,
1439                reason: Some("approved plan exit".to_string()),
1440                timestamp: OffsetDateTime::now_utc(),
1441            }))
1442            .await;
1443            self.auto_resolve_pending_tool_approvals_for_mode(current.target_mode)
1444                .await;
1445            current.target_mode
1446        } else {
1447            self.status().await.policy_mode
1448        };
1449        self.emit_plan_exit_resolved(&current, approved, resolved_mode)
1450            .await;
1451        Ok(Some(current))
1452    }
1453
1454    pub async fn resolve_tool_approval(
1455        &self,
1456        approval_id: &str,
1457        approved: bool,
1458    ) -> anyhow::Result<bool> {
1459        let pending = self.pending_tool_approvals.lock().await.remove(approval_id);
1460        let Some(pending) = pending else {
1461            return Ok(false);
1462        };
1463        self.emit(RoderEvent::ApprovalResolved(ApprovalResolved {
1464            thread_id: pending.thread_id,
1465            turn_id: pending.turn_id,
1466            approval_id: approval_id.to_string(),
1467            tool_id: pending.tool_id,
1468            tool_name: pending.tool_name,
1469            approved,
1470            timestamp: OffsetDateTime::now_utc(),
1471        }))
1472        .await;
1473        let _ = pending.tx.send(approved);
1474        Ok(true)
1475    }
1476
1477    pub async fn request_app_server_tool_approval(
1478        &self,
1479        call: ToolCall,
1480        reason: Option<String>,
1481    ) -> anyhow::Result<bool> {
1482        let approval_id = call.id.clone();
1483        let (tx, rx) = oneshot::channel();
1484        self.pending_tool_approvals.lock().await.insert(
1485            approval_id.clone(),
1486            PendingToolApproval {
1487                thread_id: call.thread_id.clone(),
1488                turn_id: call.turn_id.clone(),
1489                tool_id: call.id.clone(),
1490                tool_name: call.name.clone(),
1491                call: call.clone(),
1492                tx,
1493            },
1494        );
1495        self.emit(RoderEvent::ApprovalRequested(ApprovalRequested {
1496            thread_id: call.thread_id.clone(),
1497            turn_id: call.turn_id.clone(),
1498            approval_id,
1499            tool_id: call.id.clone(),
1500            tool_name: call.name.clone(),
1501            reason,
1502            timestamp: OffsetDateTime::now_utc(),
1503        }))
1504        .await;
1505        Ok(rx.await.unwrap_or(false))
1506    }
1507
1508    /// Completes a pending host-executed tool call (`tools/resolve`). Returns false when the
1509    /// request id is unknown, already resolved, timed out, or cancelled by a turn interrupt.
1510    pub async fn resolve_external_tool_call(
1511        &self,
1512        request_id: &str,
1513        resolution: ExternalToolResolution,
1514    ) -> anyhow::Result<bool> {
1515        let pending = self
1516            .pending_external_tool_calls
1517            .lock()
1518            .await
1519            .remove(request_id);
1520        let Some(pending) = pending else {
1521            return Ok(false);
1522        };
1523        self.emit(RoderEvent::ExternalToolCallResolved(
1524            ExternalToolCallResolved {
1525                thread_id: pending.thread_id,
1526                turn_id: pending.turn_id,
1527                request_id: request_id.to_string(),
1528                tool_id: pending.tool_id,
1529                tool_name: pending.tool_name,
1530                outcome: ExternalToolCallOutcome::Resolved,
1531                is_error: resolution.is_error,
1532                timestamp: OffsetDateTime::now_utc(),
1533            },
1534        ))
1535        .await;
1536        let _ = pending.tx.send(resolution);
1537        Ok(true)
1538    }
1539
1540    /// Drops every pending external tool call for the turn and reports them as cancelled so an
1541    /// interrupt does not leak requests waiting on `tools/resolve`.
1542    async fn cancel_pending_external_tool_calls_for_turn(&self, turn_id: &TurnId) {
1543        let cancelled = {
1544            let mut pending = self.pending_external_tool_calls.lock().await;
1545            let request_ids = pending
1546                .iter()
1547                .filter(|(_, call)| &call.turn_id == turn_id)
1548                .map(|(request_id, _)| request_id.clone())
1549                .collect::<Vec<_>>();
1550            request_ids
1551                .into_iter()
1552                .filter_map(|request_id| pending.remove(&request_id).map(|call| (request_id, call)))
1553                .collect::<Vec<_>>()
1554        };
1555        for (request_id, call) in cancelled {
1556            self.emit(RoderEvent::ExternalToolCallResolved(
1557                ExternalToolCallResolved {
1558                    thread_id: call.thread_id,
1559                    turn_id: call.turn_id,
1560                    request_id,
1561                    tool_id: call.tool_id,
1562                    tool_name: call.tool_name,
1563                    outcome: ExternalToolCallOutcome::Cancelled,
1564                    is_error: true,
1565                    timestamp: OffsetDateTime::now_utc(),
1566                },
1567            ))
1568            .await;
1569        }
1570    }
1571
1572    pub async fn resolve_user_input(
1573        &self,
1574        request_id: &str,
1575        answers: serde_json::Value,
1576    ) -> anyhow::Result<bool> {
1577        let pending = self.pending_user_inputs.lock().await.remove(request_id);
1578        let Some(pending) = pending else {
1579            return Ok(false);
1580        };
1581        self.emit(RoderEvent::UserInputResolved(UserInputResolved {
1582            thread_id: pending.thread_id,
1583            turn_id: pending.turn_id,
1584            request_id: request_id.to_string(),
1585            answers: answers.clone(),
1586            timestamp: OffsetDateTime::now_utc(),
1587        }))
1588        .await;
1589        let _ = pending.tx.send(answers);
1590        Ok(true)
1591    }
1592
1593    async fn emit_plan_exit_resolved(
1594        &self,
1595        current: &PendingPlanExit,
1596        approved: bool,
1597        resolved_mode: PolicyMode,
1598    ) {
1599        self.emit(RoderEvent::PolicyExitPlanResolved(PolicyExitPlanResolved {
1600            thread_id: current.thread_id.clone(),
1601            turn_id: current.turn_id.clone(),
1602            request_id: current.request_id.clone(),
1603            approved,
1604            target_mode: current.target_mode,
1605            resolved_mode,
1606            timestamp: OffsetDateTime::now_utc(),
1607        }))
1608        .await;
1609    }
1610
1611    pub async fn select_provider(
1612        &self,
1613        provider: String,
1614        model: Option<String>,
1615        reasoning: Option<String>,
1616    ) -> anyhow::Result<RuntimeConfig> {
1617        let next = self
1618            .preview_provider_selection(provider, model, reasoning)
1619            .await?;
1620        let mut cfg = self.config.write().await;
1621        *cfg = next;
1622        Ok(cfg.clone())
1623    }
1624
1625    pub async fn preview_provider_selection(
1626        &self,
1627        provider: String,
1628        model: Option<String>,
1629        reasoning: Option<String>,
1630    ) -> anyhow::Result<RuntimeConfig> {
1631        self.engine_for(&provider)?;
1632        let mut cfg = self.config.read().await.clone();
1633        cfg.default_provider = provider;
1634        if let Some(model) = model {
1635            cfg.default_model = model;
1636        }
1637        if let Some(reasoning) = reasoning {
1638            if reasoning == REASONING_NONE
1639                && !model_supports_reasoning(&cfg.default_model, &reasoning)
1640            {
1641                return Ok(cfg.clone());
1642            }
1643            validate_reasoning_effort(&cfg.default_model, &reasoning)?;
1644            cfg.reasoning = Some(reasoning);
1645        }
1646        Ok(cfg)
1647    }
1648
1649    pub async fn effective_reasoning(&self) -> String {
1650        let cfg = self.config.read().await;
1651        effective_reasoning_for_model(&cfg, &cfg.default_model)
1652    }
1653
1654    pub fn effective_reasoning_for_config(cfg: &RuntimeConfig) -> String {
1655        effective_reasoning_for_model(cfg, &cfg.default_model)
1656    }
1657
1658    pub async fn set_dynamic_workflow_effort(
1659        &self,
1660        effort_profile: DynamicWorkflowEffortProfile,
1661    ) -> RuntimeConfig {
1662        let mut cfg = self.config.write().await;
1663        cfg.dynamic_workflows.effort_profile = effort_profile;
1664        cfg.clone()
1665    }
1666
1667    pub async fn dynamic_workflow_trigger_decision(
1668        &self,
1669        message: &str,
1670    ) -> WorkflowTriggerDecision {
1671        let cfg = self.config.read().await;
1672        classify_workflow_trigger(message, &cfg.dynamic_workflows)
1673    }
1674
1675    pub async fn create_thread(&self, title: Option<String>) -> anyhow::Result<ThreadMetadata> {
1676        self.create_thread_with(CreateThreadRequest {
1677            title,
1678            workspace: self.workspace.display().to_string(),
1679            workspace_id: None,
1680            root_id: None,
1681            provider: None,
1682            model: None,
1683            selection_mode: None,
1684            tool_allowlist: Vec::new(),
1685            developer_instructions: None,
1686            external_tools: Vec::new(),
1687            runner: None,
1688        })
1689        .await
1690    }
1691
1692    /**
1693     * Resolves an explicit thread-runner selection into a persisted binding.
1694     * Fails fast at thread creation when the provider is missing or rejects
1695     * the destination, instead of surfacing the error on the first tool call.
1696     */
1697    async fn resolve_thread_runner_binding(
1698        &self,
1699        thread_id: &str,
1700        selection: ThreadRunnerSelection,
1701    ) -> anyhow::Result<ThreadRunnerBinding> {
1702        let provider = self
1703            .registry
1704            .remote_runner_providers
1705            .iter()
1706            .find(|provider| provider.id() == selection.provider_id)
1707            .cloned()
1708            .ok_or_else(|| {
1709                anyhow::anyhow!(
1710                    "remote runner provider {:?} is not installed",
1711                    selection.provider_id
1712                )
1713            })?;
1714        let workspace = selection.workspace.trim();
1715        anyhow::ensure!(
1716            std::path::Path::new(workspace).is_absolute(),
1717            "runner workspace must be an absolute path on the runner: {workspace:?}"
1718        );
1719        let mut read_roots = Vec::with_capacity(selection.read_roots.len());
1720        for read_root in &selection.read_roots {
1721            let trimmed = read_root.trim();
1722            anyhow::ensure!(
1723                std::path::Path::new(trimmed).is_absolute(),
1724                "runner read root must be an absolute path on the runner: {trimmed:?}"
1725            );
1726            read_roots.push(PathBuf::from(trimmed));
1727        }
1728        let destination = RunnerDestination {
1729            id: format!("thread-{thread_id}"),
1730            provider_id: selection.provider_id,
1731            config: selection.config,
1732            default_manifest: roder_api::remote_runner::RunnerManifest::default(),
1733        };
1734        provider.validate_destination(&destination).await?;
1735        Ok(ThreadRunnerBinding {
1736            destination,
1737            workspace: PathBuf::from(workspace),
1738            read_roots,
1739        })
1740    }
1741
1742    /**
1743     * Build a per-thread binding from a runtime-level destination (selected via
1744     * the TUI runner picker or config `default_destination`) when the
1745     * destination's provider advertises a default workspace. Returns `None` when
1746     * there is no destination or the provider opts out (no default workspace),
1747     * preserving the legacy behavior where such threads keep local tools.
1748     */
1749    async fn synthesize_runtime_runner_binding(
1750        &self,
1751        thread_id: &str,
1752        destination: Option<RunnerDestination>,
1753    ) -> anyhow::Result<Option<ThreadRunnerBinding>> {
1754        let Some(destination) = destination else {
1755            return Ok(None);
1756        };
1757        let Some(provider) = self
1758            .registry
1759            .remote_runner_providers
1760            .iter()
1761            .find(|provider| provider.id() == destination.provider_id)
1762        else {
1763            return Ok(None);
1764        };
1765        // An explicit workspace in the destination config wins over the
1766        // provider default; absence of both means the provider opts out.
1767        let workspace = destination
1768            .config
1769            .get("working_dir")
1770            .and_then(serde_json::Value::as_str)
1771            .map(str::to_string)
1772            .or_else(|| provider.default_workspace());
1773        let Some(workspace) = workspace else {
1774            return Ok(None);
1775        };
1776        let selection = ThreadRunnerSelection {
1777            provider_id: destination.provider_id.clone(),
1778            config: destination.config.clone(),
1779            workspace,
1780            read_roots: Vec::new(),
1781        };
1782        Ok(Some(
1783            self.resolve_thread_runner_binding(thread_id, selection)
1784                .await?,
1785        ))
1786    }
1787
1788    /// Validates a runner selection without creating a thread; the placeholder destination id only appears in error messages.
1789    pub async fn validate_thread_runner_selection(
1790        &self,
1791        selection: ThreadRunnerSelection,
1792    ) -> anyhow::Result<()> {
1793        self.resolve_thread_runner_binding("validate", selection)
1794            .await
1795            .map(|_| ())
1796    }
1797
1798    pub async fn create_thread_with(
1799        &self,
1800        req: CreateThreadRequest,
1801    ) -> anyhow::Result<ThreadMetadata> {
1802        let cfg = self.config.read().await.clone();
1803        let now = OffsetDateTime::now_utc();
1804        let workspace = validate_thread_workspace(&req.workspace)?;
1805        let provider = req.provider.unwrap_or(cfg.default_provider);
1806        let model = req.model.unwrap_or(cfg.default_model);
1807        let selection_mode = req
1808            .selection_mode
1809            .unwrap_or_else(|| ModelSelectionMode::manual(provider.clone(), model.clone(), None));
1810        let thread_id = uuid::Uuid::new_v4().to_string();
1811        let runner_binding = match req.runner {
1812            Some(selection) => Some(
1813                self.resolve_thread_runner_binding(&thread_id, selection)
1814                    .await?,
1815            ),
1816            // No explicit per-thread selection: if a runtime-level destination
1817            // is active and its provider advertises a default workspace, bind
1818            // the new thread so its coding tools route into the runner. This is
1819            // what makes a TUI/config-selected runner (e.g. Blaxel) actually
1820            // execute tools in the sandbox instead of locally. Providers that
1821            // return no default workspace stay unbound (legacy local-tools).
1822            None => {
1823                self.synthesize_runtime_runner_binding(
1824                    &thread_id,
1825                    cfg.remote_runner_destination.clone(),
1826                )
1827                .await?
1828            }
1829        };
1830        let runner_destination = runner_binding
1831            .as_ref()
1832            .map(|binding| binding.destination.clone())
1833            .or_else(|| cfg.remote_runner_destination.clone());
1834        let metadata = ThreadMetadata {
1835            thread_id,
1836            title: req.title,
1837            workspace,
1838            workspace_id: req.workspace_id,
1839            root_id: req.root_id,
1840            provider: Some(provider),
1841            model: Some(model),
1842            selection_mode: Some(selection_mode),
1843            tool_allowlist: req.tool_allowlist,
1844            developer_instructions: req.developer_instructions,
1845            external_tools: req.external_tools,
1846            runner_destination,
1847            runner_state: None,
1848            runner_binding,
1849            created_at: now,
1850            updated_at: now,
1851            message_count: 0,
1852            usage: None,
1853            parent_thread_id: None,
1854            forked_from_turn_id: None,
1855            workspace_fork: None,
1856        };
1857
1858        let metadata = if let Some(store) = &self.thread_store {
1859            store.create_thread(metadata).await?
1860        } else {
1861            metadata
1862        };
1863        self.emit(RoderEvent::ThreadCreated(ThreadCreated {
1864            thread_id: metadata.thread_id.clone(),
1865            timestamp: OffsetDateTime::now_utc(),
1866        }))
1867        .await;
1868        Ok(metadata)
1869    }
1870
1871    pub async fn list_threads(&self) -> anyhow::Result<Vec<ThreadMetadata>> {
1872        if let Some(store) = &self.thread_store {
1873            return store.list_threads().await;
1874        }
1875        Ok(Vec::new())
1876    }
1877
1878    pub async fn list_threads_page(
1879        &self,
1880        options: roder_api::thread::ThreadListOptions,
1881    ) -> anyhow::Result<roder_api::thread::ThreadListPage> {
1882        if let Some(store) = &self.thread_store {
1883            return store.list_threads_page(options).await;
1884        }
1885        Ok(roder_api::thread::ThreadListPage::default())
1886    }
1887
1888    pub async fn load_thread_metadata(
1889        &self,
1890        thread_id: &str,
1891    ) -> anyhow::Result<Option<roder_api::thread::ThreadMetadata>> {
1892        if let Some(store) = &self.thread_store {
1893            return store.load_thread_metadata(&thread_id.to_string()).await;
1894        }
1895        Ok(None)
1896    }
1897
1898    pub async fn archive_thread(&self, thread_id: &str) -> anyhow::Result<bool> {
1899        let archived = if let Some(store) = &self.thread_store {
1900            store.archive_thread(&thread_id.to_string()).await?
1901        } else {
1902            false
1903        };
1904        if archived {
1905            let workspace = self.config.read().await.workspace.clone();
1906            crate::hooks::run_lifecycle(
1907                self,
1908                &thread_id.to_string(),
1909                &SESSION_HOOK_TURN_ID.to_string(),
1910                workspace.as_deref(),
1911                "SessionEnd",
1912                Some("clear"),
1913                serde_json::json!({"reason": "archive"}),
1914            )
1915            .await;
1916            self.thread_item_cache
1917                .lock()
1918                .await
1919                .remove_thread(&thread_id.to_string());
1920            self.evict_runner_session(&thread_id.to_string()).await;
1921        }
1922        Ok(archived)
1923    }
1924
1925    pub async fn start_team(&self, req: TeamStartRequest) -> anyhow::Result<TeamState> {
1926        let cfg = self.config.read().await.clone();
1927        let workspace = self.workspace.display().to_string();
1928        let lead_thread_id = match req.lead_thread_id {
1929            Some(thread_id) => thread_id,
1930            None => {
1931                self.create_thread_with(CreateThreadRequest {
1932                    title: Some("Team lead".to_string()),
1933                    workspace: workspace.clone(),
1934                    workspace_id: None,
1935                    root_id: None,
1936                    provider: None,
1937                    model: None,
1938                    selection_mode: None,
1939                    tool_allowlist: Vec::new(),
1940                    developer_instructions: None,
1941                    external_tools: Vec::new(),
1942                    runner: None,
1943                })
1944                .await?
1945                .thread_id
1946            }
1947        };
1948        let team_id = uuid::Uuid::new_v4().to_string();
1949        let active_lead_turn_id = self.active_turn_for_thread(&lead_thread_id).await;
1950        let lead_selection = match active_lead_turn_id.as_ref() {
1951            Some(turn_id) => self
1952                .active_turn_selections
1953                .read()
1954                .await
1955                .get(turn_id)
1956                .cloned(),
1957            None => self.selection_mode_for_thread(&lead_thread_id).await?,
1958        };
1959        let lead_concrete_selection = lead_selection
1960            .as_ref()
1961            .map(ModelSelectionMode::concrete_selection);
1962        let mut lead = crate::teams::lead_member(
1963            lead_thread_id.clone(),
1964            lead_concrete_selection
1965                .as_ref()
1966                .map(|selection| selection.provider.clone())
1967                .or_else(|| Some(cfg.default_provider.clone())),
1968            lead_concrete_selection
1969                .as_ref()
1970                .map(|selection| selection.model.clone())
1971                .or_else(|| Some(cfg.default_model.clone())),
1972            cfg.policy_mode,
1973        );
1974        if let Some(turn_id) = active_lead_turn_id {
1975            lead.current_turn_id = Some(turn_id);
1976            lead.status = TeamMemberStatus::Running;
1977        }
1978        let mut members = vec![lead];
1979
1980        for (index, member) in req.members.into_iter().enumerate() {
1981            let thread = self
1982                .create_thread_with(CreateThreadRequest {
1983                    title: Some(member.name.clone()),
1984                    workspace: workspace.clone(),
1985                    workspace_id: None,
1986                    root_id: None,
1987                    provider: member.model_provider.clone(),
1988                    model: member.model.clone(),
1989                    selection_mode: None,
1990                    tool_allowlist: Vec::new(),
1991                    developer_instructions: None,
1992                    external_tools: Vec::new(),
1993                    runner: None,
1994                })
1995                .await?;
1996            let member_id = format!("member-{}", index + 1);
1997            let descriptor = crate::teams::teammate_member(
1998                member_id.clone(),
1999                member.name,
2000                thread.thread_id.clone(),
2001                member.model_provider.or(thread.provider),
2002                member.model.or(thread.model),
2003                cfg.policy_mode,
2004            );
2005            members.push(descriptor);
2006        }
2007
2008        let now = OffsetDateTime::now_utc();
2009        let team = self
2010            .teams
2011            .insert(TeamState {
2012                id: team_id.clone(),
2013                lead_thread_id: lead_thread_id.clone(),
2014                display_mode: req.display_mode,
2015                members,
2016                mailbox: Vec::new(),
2017                tasks: Vec::new(),
2018                created_at: now,
2019                updated_at: now,
2020            })
2021            .await?;
2022        self.emit(RoderEvent::TeamStarted(TeamStarted {
2023            team_id: team_id.clone(),
2024            lead_thread_id,
2025            display_mode: team.display_mode,
2026            members: team.members.clone(),
2027            tasks: team.tasks.clone(),
2028            timestamp: OffsetDateTime::now_utc(),
2029        }))
2030        .await;
2031        for member in team
2032            .members
2033            .iter()
2034            .filter(|member| member.role != roder_api::teams::TeamMemberRole::Lead)
2035        {
2036            self.emit(RoderEvent::TeamMemberStarted(TeamMemberStarted {
2037                team_id: team_id.clone(),
2038                member: member.clone(),
2039                timestamp: OffsetDateTime::now_utc(),
2040            }))
2041            .await;
2042        }
2043        Ok(team)
2044    }
2045
2046    pub async fn list_teams(&self) -> Vec<TeamState> {
2047        let active_thread_ids = self
2048            .active_turns
2049            .read()
2050            .await
2051            .values()
2052            .map(|handle| handle.thread_id.clone())
2053            .collect::<std::collections::HashSet<_>>();
2054        self.teams
2055            .list()
2056            .await
2057            .into_iter()
2058            .map(|team| runtime_local_team_view(team, &active_thread_ids))
2059            .collect()
2060    }
2061
2062    pub async fn read_team(&self, team_id: &str) -> Option<TeamState> {
2063        let active_thread_ids = self
2064            .active_turns
2065            .read()
2066            .await
2067            .values()
2068            .map(|handle| handle.thread_id.clone())
2069            .collect::<std::collections::HashSet<_>>();
2070        self.teams
2071            .get(team_id)
2072            .await
2073            .map(|team| runtime_local_team_view(team, &active_thread_ids))
2074    }
2075
2076    pub async fn start_team_member(
2077        &self,
2078        team_id: &str,
2079        req: TeamMemberStartRequest,
2080    ) -> anyhow::Result<TeamState> {
2081        self.start_team_member_with_selection(team_id, req, None)
2082            .await
2083    }
2084
2085    pub async fn message_team_member(
2086        self: &Arc<Self>,
2087        team_id: &str,
2088        member_id: &str,
2089        message: String,
2090    ) -> anyhow::Result<TurnId> {
2091        let team = self
2092            .read_team(team_id)
2093            .await
2094            .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2095        self.followup_team_member(&team.lead_thread_id, team_id, member_id, message)
2096            .await
2097    }
2098
2099    /// Queue a mailbox message without starting an idle agent. If the target is
2100    /// already running, the message is delivered at the next inference boundary.
2101    pub(crate) async fn queue_team_member_message(
2102        self: &Arc<Self>,
2103        caller_thread_id: &ThreadId,
2104        team_id: &str,
2105        member_id: &str,
2106        message: String,
2107    ) -> anyhow::Result<Option<TurnId>> {
2108        let _delivery_guard = self.agent_team_spawn_lock.lock().await;
2109        let team = self
2110            .read_team(team_id)
2111            .await
2112            .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2113        let member = team
2114            .members
2115            .iter()
2116            .find(|member| member.id == member_id)
2117            .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2118            .clone();
2119        if member.status == TeamMemberStatus::Closed {
2120            anyhow::bail!("subagent {} is closed", member.name);
2121        }
2122        let from_member_id = team
2123            .members
2124            .iter()
2125            .find(|candidate| candidate.thread_id == *caller_thread_id)
2126            .map(|candidate| candidate.id.clone());
2127        self.teams
2128            .append_mailbox_message(
2129                team_id,
2130                from_member_id,
2131                member_id.to_string(),
2132                TeamMailboxMessageKind::Message,
2133                message,
2134            )
2135            .await?;
2136        let Some(turn_id) = self.active_turn_for_thread(&member.thread_id).await else {
2137            return Ok(None);
2138        };
2139        self.deliver_pending_team_mailbox(team_id, member_id, member.thread_id, turn_id.clone())
2140            .await?;
2141        Ok(Some(turn_id))
2142    }
2143
2144    /// Deliver queued messages and start an idle agent, or steer its active turn.
2145    pub(crate) async fn followup_team_member(
2146        self: &Arc<Self>,
2147        caller_thread_id: &ThreadId,
2148        team_id: &str,
2149        member_id: &str,
2150        message: String,
2151    ) -> anyhow::Result<TurnId> {
2152        let _spawn_guard = self.agent_team_spawn_lock.lock().await;
2153        self.followup_team_member_locked(caller_thread_id, team_id, member_id, message)
2154            .await
2155    }
2156
2157    async fn followup_team_member_locked(
2158        self: &Arc<Self>,
2159        caller_thread_id: &ThreadId,
2160        team_id: &str,
2161        member_id: &str,
2162        message: String,
2163    ) -> anyhow::Result<TurnId> {
2164        let team = self
2165            .read_team(team_id)
2166            .await
2167            .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2168        let member = team
2169            .members
2170            .iter()
2171            .find(|member| member.id == member_id)
2172            .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2173            .clone();
2174        if member.status == TeamMemberStatus::Closed {
2175            anyhow::bail!("subagent {} is closed", member.name);
2176        }
2177        let from_member_id = team
2178            .members
2179            .iter()
2180            .find(|candidate| candidate.thread_id == *caller_thread_id)
2181            .map(|candidate| candidate.id.clone());
2182        self.teams
2183            .append_mailbox_message(
2184                team_id,
2185                from_member_id,
2186                member_id.to_string(),
2187                TeamMailboxMessageKind::NewTask,
2188                message,
2189            )
2190            .await?;
2191        if let Some(turn_id) = self.active_turn_for_thread(&member.thread_id).await {
2192            self.deliver_pending_team_mailbox(
2193                team_id,
2194                member_id,
2195                member.thread_id,
2196                turn_id.clone(),
2197            )
2198            .await?;
2199            return Ok(turn_id);
2200        }
2201
2202        self.ensure_codex_v2_team_capacity(&team, &member.id, None)
2203            .await?;
2204        let metadata = self.load_thread_metadata(&member.thread_id).await?;
2205        let inherited_context = self
2206            .team_member_turn_contexts
2207            .lock()
2208            .await
2209            .get(&member.thread_id)
2210            .cloned();
2211        let workspace = inherited_context
2212            .as_ref()
2213            .map(|context| context.workspace.clone())
2214            .or_else(|| metadata.as_ref().map(|metadata| metadata.workspace.clone()))
2215            .unwrap_or_else(|| self.workspace.display().to_string());
2216        let member_thread_id = member.thread_id;
2217        let turn_id = self
2218            .start_turn(StartTurnRequest {
2219                thread_id: member_thread_id.clone(),
2220                message: String::new(),
2221                images: Vec::new(),
2222                provider_override: member.model_provider,
2223                model_override: member.model,
2224                reasoning_override: metadata
2225                    .as_ref()
2226                    .and_then(|metadata| metadata.selection_mode.as_ref())
2227                    .and_then(ModelSelectionMode::reasoning)
2228                    .map(str::to_string),
2229                workspace,
2230                instructions: inherited_context
2231                    .as_ref()
2232                    .map(|context| context.instructions.clone())
2233                    .unwrap_or_else(crate::default_instructions),
2234                developer_context: inherited_context
2235                    .as_ref()
2236                    .and_then(|context| context.developer_context.clone()),
2237                task_ledger_required: false,
2238                service_tier_override: None,
2239            })
2240            .await?;
2241        Ok(turn_id)
2242    }
2243
2244    pub async fn set_team_member_policy_mode(
2245        &self,
2246        team_id: &str,
2247        member_id: &str,
2248        policy_mode: PolicyMode,
2249    ) -> anyhow::Result<TeamState> {
2250        self.teams
2251            .set_member_policy_mode(team_id, member_id, policy_mode)
2252            .await
2253    }
2254
2255    pub async fn interrupt_team_member(
2256        &self,
2257        team_id: &str,
2258        member_id: &str,
2259    ) -> anyhow::Result<Option<TurnId>> {
2260        let _interrupt_guard = self.agent_team_spawn_lock.lock().await;
2261        let team = self
2262            .read_team(team_id)
2263            .await
2264            .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2265        let member = team
2266            .members
2267            .iter()
2268            .find(|member| member.id == member_id)
2269            .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2270            .clone();
2271        if member.status != TeamMemberStatus::Running {
2272            return Ok(None);
2273        }
2274        let Some(turn_id) = member.current_turn_id.clone() else {
2275            return Ok(None);
2276        };
2277        if self
2278            .active_turn_for_thread(&member.thread_id)
2279            .await
2280            .as_ref()
2281            != Some(&turn_id)
2282        {
2283            return Ok(None);
2284        }
2285        self.interrupt_turn(member.thread_id.clone(), turn_id.clone())
2286            .await?;
2287        // Make queued mailbox work immediately eligible for the replacement turn while the
2288        // lifecycle lock still prevents a follow-up from starting. Delivery acknowledgements
2289        // are turn-owned, so a late acknowledgement from the aborted turn cannot steal a
2290        // reservation acquired by its replacement.
2291        self.teams
2292            .release_mailbox_reservations_for_turn(&turn_id)
2293            .await;
2294        self.teams
2295            .update_member(team_id, member_id, |member| {
2296                member.status = TeamMemberStatus::Interrupted;
2297                member.current_turn_id = None;
2298                member.terminal_error = None;
2299            })
2300            .await?;
2301        self.emit(RoderEvent::TeamMemberCompleted(TeamMemberCompleted {
2302            team_id: team_id.to_string(),
2303            member_id: member_id.to_string(),
2304            member_thread_id: member.thread_id,
2305            turn_id: Some(turn_id.clone()),
2306            status: TeamMemberStatus::Interrupted,
2307            final_message: member.final_message,
2308            error: None,
2309            timestamp: OffsetDateTime::now_utc(),
2310        }))
2311        .await;
2312        Ok(Some(turn_id))
2313    }
2314
2315    pub async fn close_team_member(
2316        &self,
2317        team_id: &str,
2318        member_id: &str,
2319    ) -> anyhow::Result<roder_api::teams::TeamMemberDescriptor> {
2320        let team = self
2321            .read_team(team_id)
2322            .await
2323            .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2324        let member = team
2325            .members
2326            .iter()
2327            .find(|member| member.id == member_id)
2328            .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2329            .clone();
2330        if member.role == roder_api::teams::TeamMemberRole::Lead {
2331            anyhow::bail!("team lead cannot be closed as a subagent");
2332        }
2333        let interrupted_turn_id = if member.status == TeamMemberStatus::Running {
2334            if let Some(turn_id) = member.current_turn_id.clone() {
2335                self.interrupt_turn(member.thread_id.clone(), turn_id.clone())
2336                    .await?;
2337                Some(turn_id)
2338            } else {
2339                None
2340            }
2341        } else {
2342            member.current_turn_id.clone()
2343        };
2344        let updated = self
2345            .teams
2346            .update_member(team_id, member_id, |member| {
2347                member.status = TeamMemberStatus::Closed;
2348                member.current_turn_id = None;
2349            })
2350            .await?;
2351        let closed = updated
2352            .members
2353            .iter()
2354            .find(|member| member.id == member_id)
2355            .cloned()
2356            .ok_or_else(|| anyhow::anyhow!("closed team member disappeared"))?;
2357        self.team_member_turn_contexts
2358            .lock()
2359            .await
2360            .remove(&closed.thread_id);
2361        self.emit(RoderEvent::TeamMemberCompleted(TeamMemberCompleted {
2362            team_id: team_id.to_string(),
2363            member_id: closed.id.clone(),
2364            member_thread_id: closed.thread_id.clone(),
2365            turn_id: interrupted_turn_id,
2366            status: TeamMemberStatus::Closed,
2367            final_message: closed.final_message.clone(),
2368            error: closed.terminal_error.clone(),
2369            timestamp: OffsetDateTime::now_utc(),
2370        }))
2371        .await;
2372        Ok(closed)
2373    }
2374
2375    pub async fn cleanup_team(&self, team_id: &str, force: bool) -> anyhow::Result<bool> {
2376        let Some(team) = self.read_team(team_id).await else {
2377            return Ok(false);
2378        };
2379        if !force
2380            && team
2381                .members
2382                .iter()
2383                .any(|member| member.status == TeamMemberStatus::Running)
2384        {
2385            anyhow::bail!("team {team_id:?} has active teammates; use forced cleanup");
2386        }
2387        if force {
2388            for member in team.members.iter().filter(|member| {
2389                member.role != roder_api::teams::TeamMemberRole::Lead
2390                    && member.status == TeamMemberStatus::Running
2391            }) {
2392                if let Some(turn_id) = member
2393                    .current_turn_id
2394                    .clone()
2395                    .or(self.active_turn_for_thread(&member.thread_id).await)
2396                {
2397                    let _ = self.interrupt_turn(member.thread_id.clone(), turn_id).await;
2398                }
2399            }
2400        }
2401        let removed = self.teams.remove(team_id).await?.is_some();
2402        if removed {
2403            let member_thread_ids = team
2404                .members
2405                .iter()
2406                .map(|member| member.thread_id.clone())
2407                .collect::<std::collections::HashSet<_>>();
2408            self.team_member_turn_contexts
2409                .lock()
2410                .await
2411                .retain(|thread_id, _| !member_thread_ids.contains(thread_id));
2412            self.emit(RoderEvent::TeamCleanupCompleted(TeamCleanupCompleted {
2413                team_id: team_id.to_string(),
2414                forced: force,
2415                timestamp: OffsetDateTime::now_utc(),
2416            }))
2417            .await;
2418        }
2419        Ok(removed)
2420    }
2421
2422    pub async fn effective_policy_mode_for_thread(&self, thread_id: &str) -> PolicyMode {
2423        if let Some(mode) = self.teams.policy_mode_for_thread(thread_id).await {
2424            return mode;
2425        }
2426        self.status().await.policy_mode
2427    }
2428
2429    async fn complete_team_member_turn_with_result(
2430        &self,
2431        thread_id: &ThreadId,
2432        turn_id: &TurnId,
2433        status: TeamMemberStatus,
2434        final_message: Option<String>,
2435        terminal_error: Option<String>,
2436    ) -> anyhow::Result<()> {
2437        let _delivery_guard = self.agent_team_spawn_lock.lock().await;
2438        let Some((team_id, member)) = self
2439            .teams
2440            .complete_member_turn(
2441                thread_id,
2442                turn_id,
2443                status,
2444                final_message.clone(),
2445                terminal_error.clone(),
2446            )
2447            .await?
2448        else {
2449            return Ok(());
2450        };
2451        if let Some(parent_thread_id) = member.parent_thread_id.as_ref()
2452            && let Some(team) = self.read_team(&team_id).await
2453            && let Some(parent) = team
2454                .members
2455                .iter()
2456                .find(|candidate| candidate.thread_id == *parent_thread_id)
2457        {
2458            let identity = member
2459                .agent_path
2460                .as_deref()
2461                .or(member.task_name.as_deref())
2462                .unwrap_or(&member.name);
2463            let mut report = format!("Agent {identity} finished with status {status:?}.");
2464            if let Some(message) = final_message.as_deref()
2465                && !message.trim().is_empty()
2466            {
2467                report.push_str("\n\nFinal result:\n");
2468                report.push_str(message);
2469            }
2470            if let Some(error) = terminal_error.as_deref()
2471                && !error.trim().is_empty()
2472            {
2473                report.push_str("\n\nTerminal error:\n");
2474                report.push_str(error);
2475            }
2476            let mailbox_appended = self
2477                .teams
2478                .append_mailbox_message(
2479                    &team_id,
2480                    Some(member.id.clone()),
2481                    parent.id.clone(),
2482                    TeamMailboxMessageKind::FinalAnswer,
2483                    report,
2484                )
2485                .await
2486                .is_ok();
2487            if mailbox_appended
2488                && let Some(parent_turn_id) = self.active_turn_for_thread(parent_thread_id).await
2489            {
2490                // The parent may finish between the active-turn lookup and enqueue. Its durable
2491                // mailbox entry remains pending for the next turn, while terminal observers must
2492                // still receive TeamMemberCompleted for this child.
2493                let _ = self
2494                    .deliver_pending_team_mailbox(
2495                        &team_id,
2496                        &parent.id,
2497                        parent_thread_id.clone(),
2498                        parent_turn_id,
2499                    )
2500                    .await;
2501            }
2502        }
2503        self.emit(RoderEvent::TeamMemberCompleted(TeamMemberCompleted {
2504            team_id,
2505            member_id: member.id,
2506            member_thread_id: member.thread_id,
2507            turn_id: Some(turn_id.clone()),
2508            status,
2509            final_message,
2510            error: terminal_error,
2511            timestamp: OffsetDateTime::now_utc(),
2512        }))
2513        .await;
2514        Ok(())
2515    }
2516
2517    /// Stops accepting new turns, requests interruption for every active turn,
2518    /// and waits for their runtime tasks to reach terminal cleanup up to
2519    /// `timeout`. Repeated calls are safe and continue draining the same set.
2520    pub async fn drain_active_turns(&self, timeout: std::time::Duration) -> RuntimeDrainOutcome {
2521        let started_at = tokio::time::Instant::now();
2522        let persistence_failures_before =
2523            self.lifecycle_persistence_failures.load(Ordering::Acquire);
2524        let turn_admission = self.turn_admission.lock().await;
2525        self.accepting_turns.store(false, Ordering::Release);
2526        let active = self
2527            .active_turns
2528            .read()
2529            .await
2530            .iter()
2531            .map(|(turn_id, handle)| (turn_id.clone(), handle.thread_id.clone()))
2532            .collect::<Vec<_>>();
2533        let draining = self
2534            .turn_drains
2535            .read()
2536            .await
2537            .iter()
2538            .map(|(turn_id, drain)| (turn_id.clone(), drain.thread_id.clone()))
2539            .collect::<Vec<_>>();
2540        drop(turn_admission);
2541
2542        let mut interrupted_turns = std::collections::BTreeMap::new();
2543        for (turn_id, thread_id) in active.iter().chain(draining.iter()) {
2544            interrupted_turns.insert(turn_id.clone(), thread_id.clone());
2545        }
2546        let interrupted_turn_ids = interrupted_turns.keys().cloned().collect::<Vec<_>>();
2547        for (turn_id, thread_id) in active {
2548            let _ = self
2549                .interrupt_turn_with_reason(thread_id, turn_id, TurnLifecycleReason::Shutdown)
2550                .await;
2551        }
2552
2553        let wait_for_empty = async {
2554            loop {
2555                let notified = self.active_turns_changed.notified();
2556                if self.active_turns.read().await.is_empty()
2557                    && self.turn_drains.read().await.is_empty()
2558                {
2559                    return;
2560                }
2561                notified.await;
2562            }
2563        };
2564        let outcome = if tokio::time::timeout(timeout, wait_for_empty).await.is_ok() {
2565            if self.lifecycle_persistence_failures.load(Ordering::Acquire)
2566                > persistence_failures_before
2567            {
2568                RuntimeDrainOutcome::PersistenceFailed {
2569                    interrupted_turn_ids,
2570                    remaining_turn_ids: Vec::new(),
2571                }
2572            } else {
2573                RuntimeDrainOutcome::Clean {
2574                    interrupted_turn_ids,
2575                }
2576            }
2577        } else {
2578            let active_remaining = self
2579                .active_turns
2580                .read()
2581                .await
2582                .iter()
2583                .map(|(turn_id, handle)| (turn_id.clone(), handle.drain.clone()))
2584                .collect::<Vec<_>>();
2585            let draining_remaining = self
2586                .turn_drains
2587                .read()
2588                .await
2589                .iter()
2590                .map(|(turn_id, drain)| (turn_id.clone(), drain.clone()))
2591                .collect::<Vec<_>>();
2592            let remaining = active_remaining
2593                .into_iter()
2594                .chain(draining_remaining)
2595                .collect::<std::collections::BTreeMap<_, _>>();
2596            let remaining_turn_ids = remaining.keys().cloned().collect::<Vec<_>>();
2597            for turn_id in &remaining_turn_ids {
2598                let handle = remaining
2599                    .get(turn_id)
2600                    .expect("remaining turn ID must have a drain handle");
2601                self.record_turn_lifecycle(
2602                    handle.thread_id.clone(),
2603                    turn_id.clone(),
2604                    TurnLifecycleState::InterruptRequested,
2605                    TurnCleanupState::TimedOut,
2606                    Some(TurnLifecycleReason::Shutdown),
2607                )
2608                .await;
2609            }
2610            if self.lifecycle_persistence_failures.load(Ordering::Acquire)
2611                > persistence_failures_before
2612            {
2613                RuntimeDrainOutcome::PersistenceFailed {
2614                    interrupted_turn_ids,
2615                    remaining_turn_ids,
2616                }
2617            } else {
2618                RuntimeDrainOutcome::DeadlineExceeded {
2619                    interrupted_turn_ids,
2620                    remaining_turn_ids,
2621                }
2622            }
2623        };
2624        self.record_lifecycle_drain_outcome(started_at, &outcome);
2625        outcome
2626    }
2627
2628    /// Enables turn submission after a prior drain attempt. This is intended
2629    /// for embedders that keep a runtime alive after a bounded drain timeout.
2630    pub fn resume_accepting_turns(&self) {
2631        self.accepting_turns.store(true, Ordering::Release);
2632    }
2633
2634    pub async fn turn_lifecycle_snapshot(
2635        &self,
2636        thread_id: &ThreadId,
2637    ) -> anyhow::Result<TurnLifecycleSnapshot> {
2638        let Some(store) = &self.thread_store else {
2639            return Ok(TurnLifecycleSnapshot::default());
2640        };
2641        let extension_states = store.load_extension_states(thread_id).await?;
2642        Ok(turn_lifecycle_snapshot(&extension_states))
2643    }
2644
2645    pub async fn load_thread(
2646        &self,
2647        thread_id: &ThreadId,
2648    ) -> anyhow::Result<Option<ThreadSnapshot>> {
2649        let loaded = if let Some(store) = &self.thread_store {
2650            store.load_thread(thread_id).await?
2651        } else {
2652            None
2653        };
2654        if let Some(snapshot) = loaded.as_ref() {
2655            let live_turn_ids = self
2656                .active_turns
2657                .read()
2658                .await
2659                .keys()
2660                .cloned()
2661                .chain(self.turn_drains.read().await.keys().cloned())
2662                .collect::<std::collections::HashSet<_>>();
2663            let lifecycle = turn_lifecycle_snapshot(&snapshot.extension_states);
2664            for record in lifecycle.records.into_iter().filter(|record| {
2665                record.state.requires_recovery() && !live_turn_ids.contains(&record.turn_id)
2666            }) {
2667                self.lifecycle_restart_reconciliations
2668                    .fetch_add(1, Ordering::AcqRel);
2669                self.record_turn_lifecycle(
2670                    record.thread_id,
2671                    record.turn_id,
2672                    TurnLifecycleState::RecoveryNeeded,
2673                    TurnCleanupState::Unknown,
2674                    Some(TurnLifecycleReason::RuntimeRestart),
2675                )
2676                .await;
2677            }
2678            self.emit(RoderEvent::ThreadLoaded(ThreadLoaded {
2679                thread_id: thread_id.clone(),
2680                timestamp: OffsetDateTime::now_utc(),
2681            }))
2682            .await;
2683        }
2684        Ok(loaded)
2685    }
2686
2687    pub async fn workspace_for_thread(&self, thread_id: &ThreadId) -> anyhow::Result<String> {
2688        if let Some(store) = &self.thread_store {
2689            let snapshot = store
2690                .load_thread(thread_id)
2691                .await?
2692                .ok_or_else(|| anyhow::anyhow!("thread not found: {thread_id}"));
2693            match snapshot {
2694                Ok(snapshot) => {
2695                    if let Some(metadata) = snapshot.metadata {
2696                        // Fork-backed threads fail closed before any write
2697                        // when their workspace was removed out-of-band.
2698                        if let Some(fork) = &metadata.workspace_fork
2699                            && fork.status == roder_api::forks::ForkStatus::Active
2700                            && !std::path::Path::new(&metadata.workspace).is_dir()
2701                        {
2702                            anyhow::bail!(
2703                                "workspace fork {} is missing its workspace at {}; restore it or \
2704                                 remove the fork before running turns in this thread",
2705                                fork.id,
2706                                metadata.workspace
2707                            );
2708                        }
2709                        return Ok(metadata.workspace);
2710                    }
2711                    eprintln!(
2712                        "thread metadata missing while resolving workspace for {thread_id}; falling back to runtime workspace"
2713                    );
2714                }
2715                Err(err) => {
2716                    eprintln!(
2717                        "thread missing while resolving workspace for {thread_id}: {err}; falling back to runtime workspace"
2718                    );
2719                }
2720            }
2721        }
2722        Ok(self.workspace.display().to_string())
2723    }
2724
2725    pub(crate) async fn selection_mode_for_thread(
2726        &self,
2727        thread_id: &ThreadId,
2728    ) -> anyhow::Result<Option<ModelSelectionMode>> {
2729        let Some(store) = &self.thread_store else {
2730            return Ok(None);
2731        };
2732        Ok(store
2733            .load_thread(thread_id)
2734            .await?
2735            .and_then(|snapshot| snapshot.metadata)
2736            .and_then(|metadata| {
2737                metadata
2738                    .selection_mode
2739                    .or_else(|| match (metadata.provider, metadata.model) {
2740                        (Some(provider), Some(model)) => {
2741                            Some(ModelSelectionMode::manual(provider, model, None))
2742                        }
2743                        _ => None,
2744                    })
2745            }))
2746    }
2747
2748    /// Concrete provider/model for configured-role subagents (`task`,
2749    /// `agent_swarm`). Prefers the live parent turn selection so children stay
2750    /// on SuperGrok/Grok (or whatever the lead is using) instead of process
2751    /// startup defaults such as openai/gpt-5.5.
2752    pub(crate) async fn parent_model_selection_for_subagents(
2753        &self,
2754        thread_id: &ThreadId,
2755        turn_id: &TurnId,
2756    ) -> Option<roder_api::inference::ModelSelection> {
2757        if let Some(selection) = self
2758            .active_turn_selections
2759            .read()
2760            .await
2761            .get(turn_id)
2762            .cloned()
2763        {
2764            return Some(selection.concrete_selection());
2765        }
2766        if let Ok(Some(selection)) = self.selection_mode_for_thread(thread_id).await {
2767            return Some(selection.concrete_selection());
2768        }
2769        let cfg = self.status().await;
2770        if cfg.default_provider.trim().is_empty() || cfg.default_model.trim().is_empty() {
2771            return None;
2772        }
2773        Some(roder_api::inference::ModelSelection {
2774            provider: cfg.default_provider,
2775            model: cfg.default_model,
2776        })
2777    }
2778
2779    /// Per-thread tool allowlist, developer instructions, and external tools persisted at thread creation.
2780    pub(crate) async fn thread_turn_overrides(
2781        &self,
2782        thread_id: &ThreadId,
2783    ) -> anyhow::Result<ThreadTurnOverrides> {
2784        let Some(store) = &self.thread_store else {
2785            return Ok(ThreadTurnOverrides::default());
2786        };
2787        Ok(store
2788            .load_thread_metadata(thread_id)
2789            .await?
2790            .map(|metadata| ThreadTurnOverrides {
2791                tool_allowlist: metadata.tool_allowlist,
2792                developer_instructions: metadata.developer_instructions,
2793                external_tools: metadata.external_tools,
2794            })
2795            .unwrap_or_default())
2796    }
2797
2798    pub async fn set_thread_selection_mode(
2799        &self,
2800        thread_id: &ThreadId,
2801        selection_mode: ModelSelectionMode,
2802    ) -> anyhow::Result<()> {
2803        let Some(store) = &self.thread_store else {
2804            return Ok(());
2805        };
2806        let Some(snapshot) = store.load_thread(thread_id).await? else {
2807            anyhow::bail!("thread not found: {thread_id}");
2808        };
2809        let Some(mut metadata) = snapshot.metadata else {
2810            return Ok(());
2811        };
2812        let concrete = selection_mode.concrete_selection();
2813        metadata.provider = Some(concrete.provider);
2814        metadata.model = Some(concrete.model);
2815        metadata.selection_mode = Some(selection_mode);
2816        metadata.updated_at = OffsetDateTime::now_utc();
2817        store.update_thread_metadata(metadata).await?;
2818        Ok(())
2819    }
2820
2821    async fn runner_session_for_thread(
2822        &self,
2823        thread_id: &ThreadId,
2824    ) -> anyhow::Result<Option<(RunnerDestination, Arc<dyn RemoteRunnerSession>)>> {
2825        let metadata = if let Some(store) = &self.thread_store {
2826            store.load_thread_metadata(thread_id).await?
2827        } else {
2828            None
2829        };
2830        // An explicit per-thread binding wins over the runtime-level destination.
2831        let destination = metadata
2832            .as_ref()
2833            .and_then(|metadata| metadata.runner_binding.as_ref())
2834            .map(|binding| binding.destination.clone())
2835            .or(self.config.read().await.remote_runner_destination.clone());
2836        let Some(destination) = destination else {
2837            self.evict_runner_session(thread_id).await;
2838            return Ok(None);
2839        };
2840        let provider = self
2841            .registry
2842            .remote_runner_providers
2843            .iter()
2844            .find(|provider| provider.id() == destination.provider_id)
2845            .cloned()
2846            .ok_or_else(|| {
2847                anyhow::anyhow!(
2848                    "remote runner provider {:?} is not installed",
2849                    destination.provider_id
2850                )
2851            })?;
2852        let persisted_state = metadata.and_then(|metadata| metadata.runner_state);
2853        let slot = {
2854            let mut sessions = self.runner_sessions.lock().await;
2855            match sessions.get(thread_id) {
2856                Some(slot) if slot.matches(&destination) => slot.clone(),
2857                _ => {
2858                    let slot = Arc::new(RunnerSessionSlot::empty(&destination));
2859                    sessions.insert(thread_id.clone(), slot.clone());
2860                    slot
2861                }
2862            }
2863        };
2864
2865        loop {
2866            let initialized = slot.initialized.notified();
2867            let mut state = slot.state.lock().await;
2868            match &*state {
2869                RunnerSessionSlotState::Ready(cached) => {
2870                    return Ok(Some((cached.destination.clone(), cached.session.clone())));
2871                }
2872                RunnerSessionSlotState::Failed(error) => {
2873                    return Err(anyhow::anyhow!(error.to_string()));
2874                }
2875                RunnerSessionSlotState::Initializing => {
2876                    drop(state);
2877                    initialized.await;
2878                }
2879                RunnerSessionSlotState::Empty => {
2880                    *state = RunnerSessionSlotState::Initializing;
2881                    drop(state);
2882                    self.spawn_runner_session_initialization(
2883                        thread_id.clone(),
2884                        destination.clone(),
2885                        provider.clone(),
2886                        persisted_state.clone(),
2887                        slot.clone(),
2888                    );
2889                }
2890            }
2891        }
2892    }
2893
2894    fn spawn_runner_session_initialization(
2895        &self,
2896        thread_id: ThreadId,
2897        destination: RunnerDestination,
2898        provider: Arc<dyn RemoteRunnerProvider>,
2899        persisted_state: Option<RunnerSessionState>,
2900        slot: Arc<RunnerSessionSlot>,
2901    ) {
2902        let thread_store = self.thread_store.clone();
2903        let runner_sessions = self.runner_sessions.clone();
2904        // The task intentionally outlives the turn that first requested the
2905        // workspace. Interrupting that turn must not strand the slot in
2906        // `Initializing` and deadlock every later tool or lifecycle call.
2907        drop(tokio::spawn(async move {
2908            let initialized = async {
2909                let session = if let Some(state) = persisted_state
2910                    && state.provider_id == destination.provider_id
2911                    && state.destination_id == destination.id
2912                {
2913                    match provider.resume_session(state).await {
2914                        Ok(session) => session,
2915                        Err(_) => provider.create_session(destination.clone()).await?,
2916                    }
2917                } else {
2918                    provider.create_session(destination.clone()).await?
2919                };
2920                let resolved = (destination.clone(), session.clone());
2921                // Persist before releasing the singleflight or dispatching
2922                // the first tool. A later process can rejoin even if the turn
2923                // that requested initialization has already been interrupted.
2924                Self::persist_runner_state_in_store(
2925                    thread_store.as_ref(),
2926                    &thread_id,
2927                    Some(&resolved),
2928                )
2929                .await?;
2930                Ok::<_, anyhow::Error>(CachedRunnerSession {
2931                    destination,
2932                    session,
2933                })
2934            }
2935            .await;
2936
2937            match initialized {
2938                Ok(cached) => {
2939                    *slot.state.lock().await = RunnerSessionSlotState::Ready(cached);
2940                    slot.initialized.notify_waiters();
2941                }
2942                Err(error) => {
2943                    let message: Arc<str> = Arc::from(format!("{error:#}"));
2944                    *slot.state.lock().await = RunnerSessionSlotState::Failed(message);
2945                    slot.initialized.notify_waiters();
2946                    let mut sessions = runner_sessions.lock().await;
2947                    let is_current = sessions
2948                        .get(&thread_id)
2949                        .is_some_and(|current| Arc::ptr_eq(current, &slot));
2950                    if is_current {
2951                        sessions.remove(&thread_id);
2952                    }
2953                }
2954            }
2955        }));
2956    }
2957
2958    async fn evict_runner_session(&self, thread_id: &ThreadId) {
2959        self.runner_sessions.lock().await.remove(thread_id);
2960    }
2961
2962    async fn cache_runner_session(
2963        &self,
2964        thread_id: &ThreadId,
2965        destination: RunnerDestination,
2966        session: Arc<dyn RemoteRunnerSession>,
2967    ) {
2968        let cached = CachedRunnerSession {
2969            destination,
2970            session,
2971        };
2972        self.runner_sessions.lock().await.insert(
2973            thread_id.clone(),
2974            Arc::new(RunnerSessionSlot::ready(cached)),
2975        );
2976    }
2977
2978    /// Read a thread's explicit runner binding without creating or resuming a
2979    /// session. Tool routing uses this before local previews so runner-backed
2980    /// and fail-closed hosted calls never inspect the host workspace.
2981    pub(crate) async fn runner_binding_for_thread(
2982        &self,
2983        thread_id: &ThreadId,
2984    ) -> anyhow::Result<Option<ThreadRunnerBinding>> {
2985        let Some(store) = &self.thread_store else {
2986            return Ok(None);
2987        };
2988        Ok(store
2989            .load_thread_metadata(thread_id)
2990            .await?
2991            .and_then(|metadata| metadata.runner_binding))
2992    }
2993
2994    /// Resolve a previously-read binding into a live remote workspace. The
2995    /// session remains lazy: callers invoke this only after policy approval.
2996    pub(crate) async fn remote_workspace_for_binding(
2997        &self,
2998        thread_id: &ThreadId,
2999        binding: ThreadRunnerBinding,
3000    ) -> anyhow::Result<Arc<RemoteWorkspace>> {
3001        let session = self
3002            .runner_session_for_thread(thread_id)
3003            .await?
3004            .map(|(_, session)| session)
3005            .ok_or_else(|| {
3006                anyhow::anyhow!("runner-bound thread {thread_id} has no runner session")
3007            })?;
3008        Ok(Arc::new(RemoteWorkspace {
3009            session,
3010            root: binding.workspace,
3011            read_roots: binding.read_roots,
3012        }))
3013    }
3014
3015    pub(crate) async fn runner_tool_execution_lock(
3016        &self,
3017        session: &dyn RemoteRunnerSession,
3018    ) -> Arc<RunnerToolExecutionMutex> {
3019        let state = session.state();
3020        let key = (state.provider_id, state.session_id);
3021        let mut locks = self.runner_tool_execution_locks.lock().await;
3022        if let Some(lock) = locks.get(&key).and_then(Weak::upgrade) {
3023            return lock;
3024        }
3025
3026        locks.retain(|_, lock| lock.strong_count() > 0);
3027        let lock = Arc::new(Mutex::new(()));
3028        locks.insert(key, Arc::downgrade(&lock));
3029        lock
3030    }
3031
3032    async fn persist_runner_state(
3033        &self,
3034        thread_id: &ThreadId,
3035        runner: Option<&(RunnerDestination, Arc<dyn RemoteRunnerSession>)>,
3036    ) -> anyhow::Result<()> {
3037        Self::persist_runner_state_in_store(self.thread_store.as_ref(), thread_id, runner).await
3038    }
3039
3040    async fn persist_runner_state_in_store(
3041        store: Option<&Arc<dyn ThreadStore>>,
3042        thread_id: &ThreadId,
3043        runner: Option<&(RunnerDestination, Arc<dyn RemoteRunnerSession>)>,
3044    ) -> anyhow::Result<()> {
3045        let Some((destination, session)) = runner else {
3046            return Ok(());
3047        };
3048        let Some(store) = store else {
3049            return Ok(());
3050        };
3051        let Some(snapshot) = store.load_thread(thread_id).await? else {
3052            return Ok(());
3053        };
3054        let Some(mut metadata) = snapshot.metadata else {
3055            return Ok(());
3056        };
3057        metadata.runner_destination = Some(destination.clone());
3058        metadata.runner_state = Some(session.state());
3059        metadata.updated_at = OffsetDateTime::now_utc();
3060        store.update_thread_metadata(metadata).await?;
3061        Ok(())
3062    }
3063
3064    fn remote_runner_provider_by_id(
3065        &self,
3066        provider_id: &str,
3067    ) -> Option<Arc<dyn RemoteRunnerProvider>> {
3068        self.registry
3069            .remote_runner_providers
3070            .iter()
3071            .find(|provider| provider.id() == provider_id)
3072            .cloned()
3073    }
3074
3075    /// Resolve the live runner session for a thread along with its provider,
3076    /// failing clearly when the thread is not runner-bound.
3077    async fn thread_runner_session(
3078        &self,
3079        thread_id: &ThreadId,
3080    ) -> anyhow::Result<(
3081        RunnerDestination,
3082        Arc<dyn RemoteRunnerProvider>,
3083        Arc<dyn RemoteRunnerSession>,
3084    )> {
3085        let Some((destination, session)) = self.runner_session_for_thread(thread_id).await? else {
3086            anyhow::bail!("thread {thread_id} is not bound to a remote runner");
3087        };
3088        let provider = self
3089            .remote_runner_provider_by_id(&destination.provider_id)
3090            .ok_or_else(|| {
3091                anyhow::anyhow!(
3092                    "remote runner provider {:?} is not installed",
3093                    destination.provider_id
3094                )
3095            })?;
3096        Ok((destination, provider, session))
3097    }
3098
3099    /// Pause a runner-bound thread's session toward standby and persist the
3100    /// post-pause state. Errors if the provider is not pausable.
3101    pub async fn pause_thread_runner(
3102        &self,
3103        thread_id: &ThreadId,
3104    ) -> anyhow::Result<RunnerSessionState> {
3105        let (destination, provider, session) = self.thread_runner_session(thread_id).await?;
3106        anyhow::ensure!(
3107            provider.capabilities().pausable,
3108            "remote runner provider {:?} does not support pausing",
3109            destination.provider_id
3110        );
3111        let state = session.pause().await?;
3112        self.persist_runner_state(thread_id, Some(&(destination, session)))
3113            .await?;
3114        Ok(state)
3115    }
3116
3117    /// Resume (wake) a runner-bound thread's paused session and persist state.
3118    pub async fn resume_thread_runner(
3119        &self,
3120        thread_id: &ThreadId,
3121    ) -> anyhow::Result<RunnerSessionState> {
3122        let (destination, _provider, session) = self.thread_runner_session(thread_id).await?;
3123        let state = session.resume().await?;
3124        self.persist_runner_state(thread_id, Some(&(destination, session)))
3125            .await?;
3126        Ok(state)
3127    }
3128
3129    /// Detach a runner-bound thread's session: persist the durable, rejoinable
3130    /// state and leave the remote sandbox alive. Errors if not detachable.
3131    pub async fn detach_thread_runner(
3132        &self,
3133        thread_id: &ThreadId,
3134    ) -> anyhow::Result<RunnerSessionState> {
3135        let (destination, provider, session) = self.thread_runner_session(thread_id).await?;
3136        anyhow::ensure!(
3137            provider.capabilities().detachable,
3138            "remote runner provider {:?} does not support detaching",
3139            destination.provider_id
3140        );
3141        let state = session.detach().await?;
3142        // Persist the detached state explicitly so a later turn or process
3143        // rejoins the same sandbox instead of provisioning a new one.
3144        if let Some(store) = &self.thread_store
3145            && let Some(snapshot) = store.load_thread(thread_id).await?
3146            && let Some(mut metadata) = snapshot.metadata
3147        {
3148            metadata.runner_destination = Some(destination);
3149            metadata.runner_state = Some(state.clone());
3150            metadata.updated_at = OffsetDateTime::now_utc();
3151            store.update_thread_metadata(metadata).await?;
3152        }
3153        self.evict_runner_session(thread_id).await;
3154        Ok(state)
3155    }
3156
3157    /// Rejoin a previously created sandbox from a thread's persisted runner
3158    /// state without provisioning a new one. An optional `sandbox` overrides the
3159    /// persisted sandbox name (recovery by name). Persists refreshed state.
3160    pub async fn rejoin_thread_runner(
3161        &self,
3162        thread_id: &ThreadId,
3163        sandbox: Option<String>,
3164    ) -> anyhow::Result<RunnerSessionState> {
3165        let store = self
3166            .thread_store
3167            .as_ref()
3168            .ok_or_else(|| anyhow::anyhow!("thread store is required to rejoin a runner"))?;
3169        let metadata = store
3170            .load_thread_metadata(thread_id)
3171            .await?
3172            .ok_or_else(|| anyhow::anyhow!("thread {thread_id} has no metadata"))?;
3173        let destination = metadata
3174            .runner_binding
3175            .as_ref()
3176            .map(|binding| binding.destination.clone())
3177            .or_else(|| metadata.runner_destination.clone())
3178            .ok_or_else(|| anyhow::anyhow!("thread {thread_id} is not bound to a remote runner"))?;
3179        let mut state = metadata
3180            .runner_state
3181            .clone()
3182            .ok_or_else(|| anyhow::anyhow!("thread {thread_id} has no persisted runner state"))?;
3183        if let Some(sandbox) = sandbox
3184            && let Some(object) = state.metadata.as_object_mut()
3185        {
3186            object.insert("sandbox_name".to_string(), serde_json::Value::from(sandbox));
3187        }
3188        let provider = self
3189            .remote_runner_provider_by_id(&destination.provider_id)
3190            .ok_or_else(|| {
3191                anyhow::anyhow!(
3192                    "remote runner provider {:?} is not installed",
3193                    destination.provider_id
3194                )
3195            })?;
3196        let session = provider.rejoin_session(state).await?;
3197        self.persist_runner_state(thread_id, Some(&(destination.clone(), session.clone())))
3198            .await?;
3199        self.cache_runner_session(thread_id, destination, session.clone())
3200            .await;
3201        Ok(session.state())
3202    }
3203
3204    pub(crate) async fn record_thread_usage_metadata(
3205        &self,
3206        thread_id: &ThreadId,
3207        usage: &TokenUsage,
3208    ) -> anyhow::Result<()> {
3209        if usage.is_empty() {
3210            return Ok(());
3211        }
3212        let Some(store) = &self.thread_store else {
3213            return Ok(());
3214        };
3215        let Some(snapshot) = store.load_thread(thread_id).await? else {
3216            return Ok(());
3217        };
3218        let Some(mut metadata) = snapshot.metadata else {
3219            return Ok(());
3220        };
3221        metadata
3222            .usage
3223            .get_or_insert_with(ThreadUsageMetadata::default)
3224            .add_token_usage(usage);
3225        metadata.updated_at = OffsetDateTime::now_utc();
3226        store.update_thread_metadata(metadata).await?;
3227        Ok(())
3228    }
3229
3230    pub fn start_turn(
3231        self: &Arc<Self>,
3232        mut req: StartTurnRequest,
3233    ) -> BoxFuture<'_, anyhow::Result<TurnId>> {
3234        Box::pin(async move {
3235            let _thread_admission = self.thread_admission(&req.thread_id).await;
3236            let turn_admission = self.turn_admission.lock().await;
3237            anyhow::ensure!(
3238                self.accepting_turns.load(Ordering::Acquire),
3239                "runtime is quiescing and cannot accept new turns"
3240            );
3241            req.workspace = validate_thread_workspace(&req.workspace)?;
3242            let team_member = self.teams.member_for_thread(&req.thread_id).await;
3243            let cfg = self.config.read().await.clone();
3244            let provider = req
3245                .provider_override
3246                .clone()
3247                .unwrap_or_else(|| cfg.default_provider.clone());
3248            self.engine_for(&provider)?;
3249            let turn_id = uuid::Uuid::new_v4().to_string();
3250            let mut initial_mailbox_ack = None;
3251            if let Some((team_id, member)) = &team_member {
3252                let pending = self
3253                    .teams
3254                    .reserve_pending_mailbox_messages(team_id, &member.id, &turn_id)
3255                    .await?;
3256                if !pending.is_empty()
3257                    && let Some(team) = self.read_team(team_id).await
3258                {
3259                    let mailbox = format_mailbox_messages(&team, &pending);
3260                    req.message = if req.message.trim().is_empty() {
3261                        mailbox
3262                    } else {
3263                        format!("{mailbox}\n\n[Direct task input]\n{}", req.message)
3264                    };
3265                    initial_mailbox_ack = Some(MailboxDeliveryAck {
3266                        team_id: team_id.clone(),
3267                        message_ids: pending.iter().map(|message| message.id.clone()).collect(),
3268                    });
3269                }
3270            }
3271            let (abort_handle, abort_registration) = AbortHandle::new_pair();
3272            let drain = Arc::new(TurnDrainHandle {
3273                thread_id: req.thread_id.clone(),
3274                interrupt_requested: AtomicBool::new(false),
3275                interrupt_reason: Mutex::new(None),
3276                completed: AtomicBool::new(false),
3277                completed_notify: Notify::new(),
3278            });
3279            let (steer_changed, steering) = tokio::sync::watch::channel(0);
3280            let active = ActiveTurnHandle {
3281                thread_id: req.thread_id.clone(),
3282                abort: abort_handle,
3283                steers: Arc::new(Mutex::new(Vec::new())),
3284                steer_changed,
3285                drain,
3286            };
3287            self.active_turns
3288                .write()
3289                .await
3290                .insert(turn_id.clone(), active);
3291            self.record_turn_lifecycle(
3292                req.thread_id.clone(),
3293                turn_id.clone(),
3294                TurnLifecycleState::Running,
3295                TurnCleanupState::NotRequested,
3296                None,
3297            )
3298            .await;
3299            self.active_turn_contexts.write().await.insert(
3300                turn_id.clone(),
3301                InheritedTurnContext {
3302                    workspace: req.workspace.clone(),
3303                    instructions: req.instructions.clone(),
3304                    developer_context: req.developer_context.clone(),
3305                },
3306            );
3307            if let Some((team_id, member)) = team_member {
3308                let updated = match self
3309                    .teams
3310                    .update_member(&team_id, &member.id, |member| {
3311                        member.current_turn_id = Some(turn_id.clone());
3312                        member.status = TeamMemberStatus::Running;
3313                        member.final_message = None;
3314                        member.terminal_error = None;
3315                    })
3316                    .await
3317                {
3318                    Ok(updated) => updated,
3319                    Err(error) => {
3320                        self.active_turns.write().await.remove(&turn_id);
3321                        self.active_turn_contexts.write().await.remove(&turn_id);
3322                        self.teams
3323                            .release_mailbox_reservations_for_turn(&turn_id)
3324                            .await;
3325                        return Err(error);
3326                    }
3327                };
3328                if let Some(member) = updated
3329                    .members
3330                    .into_iter()
3331                    .find(|candidate| candidate.id == member.id)
3332                {
3333                    self.emit(RoderEvent::TeamMemberStatusChanged(
3334                        TeamMemberStatusChanged {
3335                            team_id,
3336                            member_id: member.id,
3337                            member_thread_id: member.thread_id,
3338                            status: TeamMemberStatus::Running,
3339                            timestamp: OffsetDateTime::now_utc(),
3340                        },
3341                    ))
3342                    .await;
3343                }
3344            }
3345            let runtime = Arc::clone(self);
3346            let turn_req = req;
3347            let thread_id_for_task = turn_req.thread_id.clone();
3348            let turn_id_for_task = turn_id.clone();
3349            tokio::spawn(async move {
3350                let result = Abortable::new(
3351                    runtime.run_turn(
3352                        turn_req,
3353                        turn_id_for_task.clone(),
3354                        initial_mailbox_ack,
3355                        steering,
3356                    ),
3357                    abort_registration,
3358                )
3359                .await;
3360                /*
3361                 * A failed sibling in a parallel tool batch drops in-flight external tool
3362                 * futures (`try_join_all` in `route_tool_calls`), stranding their
3363                 * `pending_external_tool_calls` entries. Sweep before reporting the turn
3364                 * outcome so every `thread/toolExecutionRequested` gets a terminal
3365                 * resolution; on clean completion the map holds nothing for this turn.
3366                 */
3367                runtime
3368                    .cancel_pending_external_tool_calls_for_turn(&turn_id_for_task)
3369                    .await;
3370                let completed = matches!(&result, Ok(Ok(TurnRunOutcome::Completed)));
3371                match &result {
3372                    Ok(Err(err)) => {
3373                        let (cleanup, ownership) =
3374                            runtime.await_provider_turn_cleanup(&turn_id_for_task).await;
3375                        // This wrapper owns terminal failure emission for returned errors.
3376                        runtime
3377                            .emit(RoderEvent::TurnFailed(TurnFailed {
3378                                thread_id: thread_id_for_task.clone(),
3379                                turn_id: turn_id_for_task.clone(),
3380                                error: err.to_string(),
3381                                error_kind: err
3382                                    .downcast_ref::<roder_api::provider_error::ProviderFailure>()
3383                                    .map(|failure| failure.kind.retry_cause().to_string()),
3384                                usage: None,
3385                                timestamp: OffsetDateTime::now_utc(),
3386                            }))
3387                            .await;
3388                        runtime
3389                            .record_turn_lifecycle_with_ownership(
3390                                thread_id_for_task.clone(),
3391                                turn_id_for_task.clone(),
3392                                TurnLifecycleState::Failed,
3393                                cleanup,
3394                                Some(TurnLifecycleReason::ProviderFailure),
3395                                ownership,
3396                            )
3397                            .await;
3398                        let _ = runtime
3399                            .complete_team_member_turn_with_result(
3400                                &thread_id_for_task,
3401                                &turn_id_for_task,
3402                                TeamMemberStatus::Failed,
3403                                None,
3404                                Some(err.to_string()),
3405                            )
3406                            .await;
3407                    }
3408                    Ok(Ok(TurnRunOutcome::Stopped)) => {
3409                        let (cleanup, ownership) =
3410                            runtime.await_provider_turn_cleanup(&turn_id_for_task).await;
3411                        runtime
3412                            .record_turn_lifecycle_with_ownership(
3413                                thread_id_for_task.clone(),
3414                                turn_id_for_task.clone(),
3415                                TurnLifecycleState::Failed,
3416                                cleanup,
3417                                Some(TurnLifecycleReason::ProviderFailure),
3418                                ownership,
3419                            )
3420                            .await;
3421                        let _ = runtime
3422                            .complete_team_member_turn_with_result(
3423                                &thread_id_for_task,
3424                                &turn_id_for_task,
3425                                TeamMemberStatus::Failed,
3426                                None,
3427                                Some("turn stopped before completion".to_string()),
3428                            )
3429                            .await;
3430                    }
3431                    Err(_) => {
3432                        let reason = if let Some(handle) = runtime
3433                            .turn_drains
3434                            .read()
3435                            .await
3436                            .get(&turn_id_for_task)
3437                            .cloned()
3438                        {
3439                            handle
3440                                .interrupt_reason
3441                                .lock()
3442                                .await
3443                                .unwrap_or(TurnLifecycleReason::RuntimeFailure)
3444                        } else {
3445                            TurnLifecycleReason::RuntimeFailure
3446                        };
3447                        let (cleanup, ownership) =
3448                            runtime.await_provider_turn_cleanup(&turn_id_for_task).await;
3449                        runtime
3450                            .record_turn_lifecycle_with_ownership(
3451                                thread_id_for_task.clone(),
3452                                turn_id_for_task.clone(),
3453                                TurnLifecycleState::Interrupted,
3454                                cleanup,
3455                                Some(reason),
3456                                ownership,
3457                            )
3458                            .await;
3459                        runtime
3460                            .emit(RoderEvent::TurnInterrupted(TurnInterrupted {
3461                                thread_id: thread_id_for_task.clone(),
3462                                turn_id: turn_id_for_task.clone(),
3463                                timestamp: OffsetDateTime::now_utc(),
3464                            }))
3465                            .await;
3466                        let _ = runtime
3467                            .complete_team_member_turn_with_result(
3468                                &thread_id_for_task,
3469                                &turn_id_for_task,
3470                                TeamMemberStatus::Interrupted,
3471                                None,
3472                                None,
3473                            )
3474                            .await;
3475                    }
3476                    Ok(Ok(TurnRunOutcome::Completed)) => {}
3477                }
3478                runtime
3479                    .teams
3480                    .release_mailbox_reservations_for_turn(&turn_id_for_task)
3481                    .await;
3482                runtime.active_turns.write().await.remove(&turn_id_for_task);
3483                if let Some(drain) = runtime.turn_drains.write().await.remove(&turn_id_for_task) {
3484                    drain.completed.store(true, Ordering::Release);
3485                    drain.completed_notify.notify_waiters();
3486                }
3487                runtime
3488                    .active_turn_selections
3489                    .write()
3490                    .await
3491                    .remove(&turn_id_for_task);
3492                runtime
3493                    .active_turn_contexts
3494                    .write()
3495                    .await
3496                    .remove(&turn_id_for_task);
3497                if !completed {
3498                    // Completion paths above consume registered provider cleanup
3499                    // handles. A setup failure before an engine can stream has no
3500                    // such handle; this is a harmless final defensive sweep.
3501                    let _ = runtime.provider_turn_cleanups.lock().map(|mut cleanups| {
3502                        cleanups.remove(&turn_id_for_task);
3503                    });
3504                }
3505                runtime.active_turns_changed.notify_waiters();
3506                if completed {
3507                    let _ = runtime
3508                        .continue_active_goal_after_turn(thread_id_for_task)
3509                        .await;
3510                }
3511            });
3512            drop(turn_admission);
3513            Ok(turn_id)
3514        })
3515    }
3516
3517    pub(crate) async fn has_active_turn_for_thread(&self, thread_id: &ThreadId) -> bool {
3518        self.active_turns
3519            .read()
3520            .await
3521            .values()
3522            .any(|handle| &handle.thread_id == thread_id)
3523    }
3524
3525    async fn ensure_codex_v2_team_capacity(
3526        &self,
3527        team: &TeamState,
3528        resuming_member_id: &str,
3529        candidate_model: Option<&str>,
3530    ) -> anyhow::Result<()> {
3531        if !is_codex_v2_team(team)
3532            && !candidate_model
3533                .is_some_and(|model| model_supports_reasoning_effort(model, REASONING_ULTRA))
3534        {
3535            return Ok(());
3536        }
3537        let active_thread_ids = self
3538            .active_turns
3539            .read()
3540            .await
3541            .values()
3542            .map(|handle| handle.thread_id.clone())
3543            .collect::<std::collections::HashSet<_>>();
3544        let resident_threads = 1 + team
3545            .members
3546            .iter()
3547            .filter(|member| {
3548                member.role != roder_api::teams::TeamMemberRole::Lead
3549                    && member.id != resuming_member_id
3550                    && active_thread_ids.contains(&member.thread_id)
3551            })
3552            .count();
3553        anyhow::ensure!(
3554            resident_threads < codex_v2::CODEX_V2_MAX_RESIDENT_TEAM_THREADS,
3555            "agent thread limit reached for this Codex V2 team: maximum {} resident threads (the lead plus 3 running subagents)",
3556            codex_v2::CODEX_V2_MAX_RESIDENT_TEAM_THREADS
3557        );
3558        Ok(())
3559    }
3560
3561    /// Number of currently running turns across all threads. Hosted runtime
3562    /// pools use this to avoid evicting tenants with active work.
3563    pub async fn active_turn_count(&self) -> usize {
3564        self.active_turns.read().await.len()
3565    }
3566
3567    pub async fn active_turn_for_thread(&self, thread_id: &ThreadId) -> Option<TurnId> {
3568        self.active_turns
3569            .read()
3570            .await
3571            .iter()
3572            .find_map(|(turn_id, handle)| (&handle.thread_id == thread_id).then(|| turn_id.clone()))
3573    }
3574
3575    pub async fn thread_activity(&self, thread_id: &ThreadId) -> ThreadActivity {
3576        let active_turn_id = self.active_turn_for_thread(thread_id).await;
3577        let draining_turn_id = if active_turn_id.is_none() {
3578            self.turn_drains
3579                .read()
3580                .await
3581                .iter()
3582                .find_map(|(turn_id, drain)| {
3583                    (&drain.thread_id == thread_id).then(|| turn_id.clone())
3584                })
3585        } else {
3586            None
3587        };
3588        let Some(active_turn_id) = active_turn_id.or(draining_turn_id) else {
3589            return ThreadActivity::default();
3590        };
3591
3592        let mut active_flags = Vec::new();
3593        {
3594            let pending_approvals = self.pending_tool_approvals.lock().await;
3595            if pending_approvals
3596                .values()
3597                .any(|pending| &pending.thread_id == thread_id && pending.turn_id == active_turn_id)
3598            {
3599                active_flags.push("approvalRequired".to_string());
3600            }
3601        }
3602        {
3603            let pending_inputs = self.pending_user_inputs.lock().await;
3604            if pending_inputs
3605                .values()
3606                .any(|pending| &pending.thread_id == thread_id && pending.turn_id == active_turn_id)
3607            {
3608                active_flags.push("userInputRequired".to_string());
3609            }
3610        }
3611        {
3612            let pending_external = self.pending_external_tool_calls.lock().await;
3613            if pending_external
3614                .values()
3615                .any(|pending| &pending.thread_id == thread_id && pending.turn_id == active_turn_id)
3616            {
3617                active_flags.push("externalToolPending".to_string());
3618            }
3619        }
3620        let interrupting = self
3621            .active_turns
3622            .read()
3623            .await
3624            .get(&active_turn_id)
3625            .is_some_and(|handle| handle.drain.interrupt_requested.load(Ordering::Acquire))
3626            || self
3627                .turn_drains
3628                .read()
3629                .await
3630                .get(&active_turn_id)
3631                .is_some_and(|drain| drain.interrupt_requested.load(Ordering::Acquire));
3632        if interrupting {
3633            active_flags.push("interrupting".to_string());
3634        }
3635        if self.pending_plan_exit().await.is_some_and(|pending| {
3636            &pending.thread_id == thread_id && pending.turn_id == active_turn_id
3637        }) {
3638            active_flags.push("planExitRequired".to_string());
3639        }
3640
3641        ThreadActivity {
3642            active_turn_id: Some(active_turn_id),
3643            active_flags,
3644        }
3645    }
3646
3647    pub async fn interrupt_turn(&self, thread_id: ThreadId, turn_id: TurnId) -> anyhow::Result<()> {
3648        self.interrupt_turn_with_reason(thread_id, turn_id, TurnLifecycleReason::UserInterrupt)
3649            .await
3650    }
3651
3652    async fn interrupt_turn_with_reason(
3653        &self,
3654        thread_id: ThreadId,
3655        turn_id: TurnId,
3656        reason: TurnLifecycleReason,
3657    ) -> anyhow::Result<()> {
3658        let handle = self.active_turns.write().await.remove(&turn_id);
3659        if let Some(handle) = handle {
3660            if handle
3661                .drain
3662                .interrupt_requested
3663                .swap(true, Ordering::AcqRel)
3664            {
3665                return Ok(());
3666            }
3667            *handle.drain.interrupt_reason.lock().await = Some(reason);
3668            self.turn_drains
3669                .write()
3670                .await
3671                .insert(turn_id.clone(), handle.drain.clone());
3672            let ownership = self.provider_cleanup_ownership(&turn_id);
3673            self.record_turn_lifecycle_with_ownership(
3674                thread_id.clone(),
3675                turn_id.clone(),
3676                TurnLifecycleState::InterruptRequested,
3677                TurnCleanupState::Requested,
3678                Some(reason),
3679                ownership,
3680            )
3681            .await;
3682            handle.abort.abort();
3683            self.active_turns_changed.notify_waiters();
3684        }
3685        self.active_turn_selections.write().await.remove(&turn_id);
3686        self.cancel_pending_external_tool_calls_for_turn(&turn_id)
3687            .await;
3688        Ok(())
3689    }
3690
3691    pub async fn steer_turn(
3692        &self,
3693        thread_id: ThreadId,
3694        turn_id: TurnId,
3695        message: String,
3696        images: Vec<InputImage>,
3697    ) -> anyhow::Result<()> {
3698        self.enqueue_turn_steer(thread_id, turn_id, message, images, None)
3699            .await
3700    }
3701
3702    pub(crate) async fn has_pending_turn_steers(&self, turn_id: &TurnId) -> bool {
3703        let active = self.active_turns.read().await.get(turn_id).cloned();
3704        let Some(active) = active else {
3705            return false;
3706        };
3707        let has_pending = !active.steers.lock().await.is_empty();
3708        has_pending
3709    }
3710
3711    async fn steer_turn_with_mailbox_ack(
3712        &self,
3713        thread_id: ThreadId,
3714        turn_id: TurnId,
3715        message: String,
3716        message_ids: Vec<String>,
3717        team_id: TeamId,
3718    ) -> anyhow::Result<()> {
3719        self.enqueue_turn_steer(
3720            thread_id,
3721            turn_id,
3722            message,
3723            Vec::new(),
3724            Some(MailboxDeliveryAck {
3725                team_id,
3726                message_ids,
3727            }),
3728        )
3729        .await
3730    }
3731
3732    async fn deliver_pending_team_mailbox(
3733        &self,
3734        team_id: &str,
3735        member_id: &str,
3736        member_thread_id: ThreadId,
3737        turn_id: TurnId,
3738    ) -> anyhow::Result<()> {
3739        let pending = self
3740            .teams
3741            .reserve_pending_mailbox_messages(team_id, member_id, &turn_id)
3742            .await?;
3743        if pending.is_empty() {
3744            return Ok(());
3745        }
3746        let message_ids = pending
3747            .iter()
3748            .map(|message| message.id.clone())
3749            .collect::<Vec<_>>();
3750        let team = self
3751            .read_team(team_id)
3752            .await
3753            .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
3754        if let Err(error) = self
3755            .steer_turn_with_mailbox_ack(
3756                member_thread_id,
3757                turn_id.clone(),
3758                format_mailbox_messages(&team, &pending),
3759                message_ids.clone(),
3760                team_id.to_string(),
3761            )
3762            .await
3763        {
3764            self.teams
3765                .release_mailbox_reservations(&turn_id, &message_ids)
3766                .await;
3767            return Err(error);
3768        }
3769        Ok(())
3770    }
3771
3772    async fn enqueue_turn_steer(
3773        &self,
3774        thread_id: ThreadId,
3775        turn_id: TurnId,
3776        message: String,
3777        images: Vec<InputImage>,
3778        mailbox_ack: Option<MailboxDeliveryAck>,
3779    ) -> anyhow::Result<()> {
3780        let message = message.trim().to_string();
3781        if message.is_empty() && images.is_empty() {
3782            return Ok(());
3783        }
3784
3785        let Some(active) = self.active_turns.read().await.get(&turn_id).cloned() else {
3786            anyhow::bail!("no active turn to steer");
3787        };
3788        anyhow::ensure!(
3789            active.thread_id == thread_id,
3790            "turn does not belong to thread"
3791        );
3792        {
3793            let mut steers = active.steers.lock().await;
3794            steers.push(QueuedTurnSteer {
3795                message: UserMessage::with_images(message.clone(), images),
3796                mailbox_ack,
3797            });
3798            active
3799                .steer_changed
3800                .send_modify(|generation| *generation = generation.wrapping_add(1));
3801        }
3802        self.emit(RoderEvent::TurnSteered(TurnSteered {
3803            thread_id,
3804            turn_id,
3805            message,
3806            timestamp: OffsetDateTime::now_utc(),
3807        }))
3808        .await;
3809        Ok(())
3810    }
3811
3812    pub async fn tool_specs(&self) -> Vec<roder_api::tools::ToolSpec> {
3813        let cfg = self.config.read().await;
3814        let model_profile =
3815            model_profile_for_provider_model(&cfg, &cfg.default_provider, &cfg.default_model);
3816        self.filtered_tool_specs(&cfg, &cfg.default_model, model_profile.as_ref(), &[], &[])
3817    }
3818
3819    pub fn subagent_definitions(&self) -> Vec<SubagentDefinition> {
3820        self.registry
3821            .subagent_dispatchers
3822            .iter()
3823            .flat_map(|dispatcher| dispatcher.definitions())
3824            .collect()
3825    }
3826
3827    async fn run_turn(
3828        self: &Arc<Self>,
3829        req: StartTurnRequest,
3830        turn_id: TurnId,
3831        initial_mailbox_ack: Option<MailboxDeliveryAck>,
3832        mut steering: tokio::sync::watch::Receiver<u64>,
3833    ) -> anyhow::Result<TurnRunOutcome> {
3834        let turn_started_at = OffsetDateTime::now_utc();
3835        self.emit(RoderEvent::TurnStarted(TurnStarted {
3836            thread_id: req.thread_id.clone(),
3837            turn_id: turn_id.clone(),
3838            runtime_profile: self.config.read().await.runtime_profile,
3839            timestamp: turn_started_at,
3840        }))
3841        .await;
3842        self.persist_turn_item(
3843            &req.thread_id,
3844            &turn_id,
3845            &TranscriptItem::UserMessage(UserMessage::with_images(
3846                req.message.clone(),
3847                req.images.clone(),
3848            )),
3849        )
3850        .await?;
3851        crate::hooks::run_lifecycle(
3852            self,
3853            &req.thread_id,
3854            &turn_id,
3855            Some(&req.workspace),
3856            "UserPromptSubmit",
3857            None,
3858            serde_json::json!({"prompt": req.message}),
3859        )
3860        .await;
3861        if let Some(ack) = initial_mailbox_ack {
3862            self.teams
3863                .mark_mailbox_messages_delivered(&ack.team_id, &turn_id, &ack.message_ids)
3864                .await?;
3865        }
3866
3867        let mut cfg = self.config.read().await.clone();
3868        let runtime_profile = cfg.runtime_profile;
3869        let turn_deadline = turn_deadline_for_config(&cfg);
3870        let deadline_finalization_reserve =
3871            crate::deadline_policy::finalization_reserve_seconds(cfg.turn_deadline_seconds);
3872        let selection_mode = self.selection_mode_for_thread(&req.thread_id).await?;
3873        let concrete_selection = selection_mode
3874            .as_ref()
3875            .map(ModelSelectionMode::concrete_selection);
3876        let default_provider = req
3877            .provider_override
3878            .clone()
3879            .or_else(|| {
3880                concrete_selection
3881                    .as_ref()
3882                    .map(|selection| selection.provider.clone())
3883            })
3884            .unwrap_or(cfg.default_provider.clone());
3885        let default_model = req
3886            .model_override
3887            .clone()
3888            .or_else(|| {
3889                concrete_selection
3890                    .as_ref()
3891                    .map(|selection| selection.model.clone())
3892            })
3893            .unwrap_or(cfg.default_model.clone());
3894        if let Some(reasoning) = req.reasoning_override.as_deref().or_else(|| {
3895            selection_mode
3896                .as_ref()
3897                .and_then(ModelSelectionMode::reasoning)
3898        }) {
3899            validate_reasoning_effort(&default_model, reasoning)?;
3900            cfg.reasoning = Some(reasoning.to_string());
3901        }
3902        let turn_has_concrete_model_override =
3903            req.provider_override.is_some() || req.model_override.is_some();
3904        let (turn_inference_router, turn_inference_router_profile) = match &selection_mode {
3905            Some(ModelSelectionMode::Auto {
3906                router_id, profile, ..
3907            }) if !turn_has_concrete_model_override => (
3908                RuntimeInferenceRouterConfig {
3909                    enabled: true,
3910                    router_id: Some(router_id.clone()),
3911                },
3912                profile.clone(),
3913            ),
3914            _ => (RuntimeInferenceRouterConfig::disabled(), None),
3915        };
3916        let mut provider = default_provider.clone();
3917        let mut model = default_model.clone();
3918        let mut model_profile = model_profile_for_provider_model(&cfg, &provider, &model);
3919        let workspace = req.workspace.clone();
3920        let compaction_generation_at_start = self.compaction_generation(&req.thread_id);
3921        let mut transcript = self.transcript_for_turn(&req, &turn_id, &model).await?;
3922        let mut compacted_this_turn =
3923            self.compaction_generation(&req.thread_id) != compaction_generation_at_start;
3924        let effective_policy_mode = self.effective_policy_mode_for_thread(&req.thread_id).await;
3925        let agent_swarm_mode_active = self
3926            .effective_agent_swarm_mode_for_thread(&req.thread_id)
3927            .await;
3928        let ultra_mode_active = self.effective_ultra_mode_for_thread(&req.thread_id).await;
3929        let thread_overrides = self.thread_turn_overrides(&req.thread_id).await?;
3930        let mut final_assistant_text = String::new();
3931        let mut final_provider_metadata = None;
3932        let mut exhausted_tool_rounds = true;
3933        let mut verification_gate =
3934            VerificationGateState::new(req.message.clone(), runtime_profile);
3935        let mut speed_policy = SpeedPolicyState::default();
3936        let mut reliability = TurnReliabilityState::default();
3937        let mut turn_usage = TokenUsage::default();
3938        // Overwritten on every inference step's Completed event so only the
3939        // terminal step's stop reason survives (mid-turn tool_use steps must
3940        // not leak onto turn/completed).
3941        let mut turn_finish_reason: Option<String> = None;
3942        let mut deadline_finalization_requested = false;
3943        let mut deadline_scoreable_completion_requested = false;
3944        let mut task_ledger_completion_reminders = 0_u8;
3945        let mut task_ledger_scoreable_checkpoints = 0_u8;
3946        let mut empty_tool_call_nudges_used = 0_u32;
3947        let mut provider_stream_retry_attempts = 0_u32;
3948        let mut context_limit_recovery_attempts = 0_u32;
3949        let mut routing_candidates = None;
3950        let routing_transcript_start = transcript.len().saturating_sub(1);
3951        let mut routing_escalations = 0_u32;
3952        let mut model_switch_summary_selection = None::<ModelSelection>;
3953
3954        'tool_rounds: for round_index in 0..MAX_TOOL_ROUNDS_PER_TURN {
3955            if let Some(deadline) = turn_deadline
3956                && deadline_expired(deadline)
3957            {
3958                self.fail_turn_due_to_deadline(&req.thread_id, &turn_id, deadline, &transcript)
3959                    .await?;
3960                return Ok(TurnRunOutcome::Stopped);
3961            }
3962            let steers = self.drain_turn_steers(&turn_id, &mut steering).await;
3963            self.append_steers(&req, &turn_id, &mut transcript, steers)
3964                .await?;
3965            if runtime_profile == RuntimeProfile::Eval
3966                && let Some(remaining) = crate::deadline_policy::should_start_finalization(
3967                    turn_deadline,
3968                    deadline_finalization_reserve,
3969                    deadline_finalization_requested || deadline_scoreable_completion_requested,
3970                )
3971            {
3972                if req.task_ledger_required
3973                    && task_ledger_completion_reminders < TASK_LEDGER_COMPLETION_REMINDER_LIMIT
3974                    && let Some(prompt) = task_ledger_completion_prompt(&transcript)
3975                {
3976                    task_ledger_completion_reminders += 1;
3977                    deadline_scoreable_completion_requested = true;
3978                    let item = TranscriptItem::UserMessage(UserMessage::text(
3979                        task_ledger_deadline_completion_prompt(
3980                            remaining,
3981                            deadline_finalization_reserve,
3982                            &prompt,
3983                        ),
3984                    ));
3985                    self.persist_turn_item(&req.thread_id, &turn_id, &item)
3986                        .await?;
3987                    transcript.push(item);
3988                    continue 'tool_rounds;
3989                } else {
3990                    self.start_deadline_finalization(
3991                        &req.thread_id,
3992                        &turn_id,
3993                        &mut transcript,
3994                        remaining,
3995                    )
3996                    .await?;
3997                    deadline_finalization_requested = true;
3998                }
3999            }
4000            if runtime_profile == RuntimeProfile::Eval
4001                && req.task_ledger_required
4002                && !deadline_finalization_requested
4003                && task_ledger_scoreable_checkpoints < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
4004                && let Some(remaining) = deadline_remaining_seconds(turn_deadline)
4005                && remaining <= TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS
4006                && remaining > deadline_finalization_reserve
4007                && let Some(prompt) = task_ledger_completion_prompt(&transcript)
4008            {
4009                task_ledger_scoreable_checkpoints += 1;
4010                let item = TranscriptItem::UserMessage(UserMessage::text(
4011                    task_ledger_scoreable_checkpoint_prompt(remaining, &prompt),
4012                ));
4013                self.persist_turn_item(&req.thread_id, &turn_id, &item)
4014                    .await?;
4015                transcript.push(item);
4016                continue 'tool_rounds;
4017            }
4018            if turn_inference_router.is_active() && routing_candidates.is_none() {
4019                routing_candidates =
4020                    Some(collect_inference_routing_candidates(&self.registry).await);
4021            }
4022            let routing_tools_model = model.clone();
4023            let routing_tools = self.filtered_tool_specs(
4024                &cfg,
4025                &model,
4026                model_profile.as_ref(),
4027                &thread_overrides.tool_allowlist,
4028                &thread_overrides.external_tools,
4029            );
4030            let prior_failures =
4031                transcript_failure_count_since(&transcript, routing_transcript_start)
4032                    .max(reliability.tool_failure_count())
4033                    .saturating_add(provider_stream_retry_attempts);
4034            let routing_selection = route_inference_selection(
4035                &self.registry,
4036                &turn_inference_router,
4037                InferenceRoutingRequest {
4038                    thread_id: &req.thread_id,
4039                    turn_id: &turn_id,
4040                    round_index: round_index as u32,
4041                    runtime_profile,
4042                    phase: speed_policy.phase(),
4043                    profile: turn_inference_router_profile.as_deref(),
4044                    default_selection: ModelSelection {
4045                        provider: default_provider.clone(),
4046                        model: default_model.clone(),
4047                    },
4048                    transcript: &transcript,
4049                    tools: &routing_tools,
4050                    candidates: routing_candidates.as_deref(),
4051                    prior_failures,
4052                    prior_escalations: routing_escalations,
4053                },
4054            )
4055            .await;
4056            if let Some(decision) = routing_selection.decision.clone() {
4057                if matches!(decision.outcome, InferenceRoutingOutcome::Escalated) {
4058                    routing_escalations = routing_escalations.saturating_add(1);
4059                }
4060                self.emit(RoderEvent::InferenceRoutingDecision(
4061                    InferenceRoutingDecisionEvent {
4062                        thread_id: req.thread_id.clone(),
4063                        turn_id: turn_id.clone(),
4064                        round_index: round_index as u32,
4065                        default_selection: ModelSelection {
4066                            provider: default_provider.clone(),
4067                            model: default_model.clone(),
4068                        },
4069                        selected_selection: routing_selection.selection.clone(),
4070                        decision,
4071                        timestamp: OffsetDateTime::now_utc(),
4072                    },
4073                ))
4074                .await;
4075            }
4076            provider = routing_selection.selection.provider.clone();
4077            model = routing_selection.selection.model.clone();
4078            let engine = self.engine_for(&provider)?;
4079            let capabilities = engine.capabilities();
4080            model_profile = model_profile_for_provider_model(&cfg, &provider, &model);
4081            let tools = if capabilities.tool_calls {
4082                if model == routing_tools_model {
4083                    routing_tools.clone()
4084                } else {
4085                    self.filtered_tool_specs(
4086                        &cfg,
4087                        &model,
4088                        model_profile.as_ref(),
4089                        &thread_overrides.tool_allowlist,
4090                        &thread_overrides.external_tools,
4091                    )
4092                }
4093            } else {
4094                Vec::new()
4095            };
4096            let parallel_tool_calls = parallel_tool_calls_for_model(&cfg, &model);
4097            let tool_choice = if tools.is_empty() {
4098                ToolChoice::None
4099            } else {
4100                ToolChoice::Auto
4101            };
4102            let summary_selection = ModelSelection {
4103                provider: provider.clone(),
4104                model: model.clone(),
4105            };
4106            if model_switch_summary_selection.as_ref() != Some(&summary_selection) {
4107                if let Some(summary) = model_switch_summary(
4108                    &transcript,
4109                    model_profile.as_ref(),
4110                    &provider,
4111                    &model,
4112                    &tools,
4113                ) {
4114                    let item = TranscriptItem::UserMessage(UserMessage::text(summary));
4115                    self.persist_turn_item(&req.thread_id, &turn_id, &item)
4116                        .await?;
4117                    transcript.push(item);
4118                }
4119                model_switch_summary_selection = Some(summary_selection);
4120            }
4121
4122            if !capabilities.image_input && transcript_has_images(&transcript) {
4123                self.fail_turn_with_error(
4124                    &req.thread_id,
4125                    &turn_id,
4126                    format!("provider {provider} does not support image input"),
4127                )
4128                .await?;
4129                return Ok(TurnRunOutcome::Stopped);
4130            }
4131            let compaction_generation_before = self.compaction_generation(&req.thread_id);
4132            transcript = self
4133                .compact_transcript_if_needed(
4134                    &req.thread_id,
4135                    &turn_id,
4136                    &provider,
4137                    &model,
4138                    transcript,
4139                    self.compaction_options_for_turn(&req.thread_id, !compacted_this_turn),
4140                )
4141                .await?;
4142            compacted_this_turn |=
4143                self.compaction_generation(&req.thread_id) != compaction_generation_before;
4144
4145            let speed_policy_decision =
4146                speed_policy.decision(runtime_profile, &model, &cfg.speed_policy);
4147            let request_reasoning = reasoning_from_decision(
4148                speed_policy_decision.as_ref(),
4149                routing_selection
4150                    .reasoning
4151                    .clone()
4152                    .unwrap_or_else(|| reasoning_for_model(&cfg, &model)),
4153            );
4154            self.active_turn_selections.write().await.insert(
4155                turn_id.clone(),
4156                ModelSelectionMode::manual(
4157                    provider.clone(),
4158                    model.clone(),
4159                    request_reasoning.level.clone(),
4160                ),
4161            );
4162            if let Some(limit) = reliability.record_model_call(
4163                &cfg.reliability,
4164                runtime_profile == RuntimeProfile::Interactive,
4165            ) {
4166                self.fail_turn_due_to_reliability_limit(
4167                    &req.thread_id,
4168                    &turn_id,
4169                    &provider,
4170                    &model,
4171                    limit,
4172                    &transcript,
4173                )
4174                .await?;
4175                return Ok(TurnRunOutcome::Stopped);
4176            }
4177            self.emit(RoderEvent::InferenceStarted(InferenceStarted {
4178                thread_id: req.thread_id.clone(),
4179                turn_id: turn_id.clone(),
4180                engine_id: engine.id(),
4181                model: ModelSelection {
4182                    provider: provider.clone(),
4183                    model: model.clone(),
4184                },
4185                reasoning: request_reasoning.clone(),
4186                speed_policy: speed_policy_decision.clone(),
4187                deadline_remaining_seconds: deadline_remaining_seconds(turn_deadline),
4188                timestamp: OffsetDateTime::now_utc(),
4189            }))
4190            .await;
4191
4192            let mut instructions = req.instructions.clone();
4193            if let Some(extra) = &thread_overrides.developer_instructions {
4194                instructions = apply_thread_developer_instructions(instructions, extra);
4195            }
4196            if let Some(context) = req.developer_context.as_deref() {
4197                instructions = apply_turn_developer_context(instructions, context);
4198            }
4199            let mut instructions = apply_runtime_profile(instructions, runtime_profile);
4200            if let Some(profile) = &model_profile {
4201                instructions = apply_model_instruction_overlay(instructions, profile);
4202            }
4203            if req.task_ledger_required
4204                && runtime_profile == RuntimeProfile::Eval
4205                && !transcript_has_task_ledger(&transcript)
4206            {
4207                instructions = apply_task_ledger_required(instructions);
4208            }
4209            if effective_policy_mode == PolicyMode::Plan {
4210                instructions = apply_plan_mode(instructions);
4211            }
4212            if agent_swarm_mode_active {
4213                instructions = apply_agent_swarm_mode(instructions);
4214            }
4215            // Ultra mode (any model) enables proactive multi-agent policy.
4216            // Codex Sol/Terra Ultra effort still does too without requiring the
4217            // mode flag. Models that advertise Ultra effort keep the
4218            // explicit-request-only policy on lower efforts when ultra mode is
4219            // off; other models get multi-agent policy only when ultra mode is on.
4220            let model_has_ultra_effort = model_supports_reasoning_effort(&model, REASONING_ULTRA);
4221            let effort_is_ultra = request_reasoning.level.as_deref() == Some(REASONING_ULTRA);
4222            let proactive_multi_agent = ultra_mode_active || effort_is_ultra;
4223            if ultra_mode_active || model_has_ultra_effort {
4224                instructions = apply_codex_multi_agent_mode(instructions, proactive_multi_agent);
4225            }
4226            instructions = self
4227                .goals
4228                .apply_goal_instructions(&req.thread_id, instructions)
4229                .await?;
4230            instructions =
4231                apply_parallel_web_tools(instructions, tools.iter().map(|spec| spec.name.as_str()));
4232            let mut request_metadata = serde_json::json!({});
4233            if let Some(decision) = &speed_policy_decision {
4234                request_metadata["speedPolicy"] = serde_json::json!(decision);
4235            }
4236            if let Some(decision) = routing_selection.decision.as_ref() {
4237                request_metadata["inferenceRouting"] = serde_json::json!(decision);
4238            }
4239            if let Some(remaining) = deadline_remaining_seconds(turn_deadline) {
4240                request_metadata["deadlineRemainingSeconds"] = serde_json::json!(remaining);
4241            }
4242            if let Some(profile) = &model_profile {
4243                request_metadata["modelProfile"] = serde_json::json!({
4244                    "model": profile.model,
4245                    "providerFamily": profile.provider_family,
4246                    "editTool": profile.edit_tool,
4247                    "schemaPolicy": profile.schema_policy,
4248                    "instructionOverlay": profile.instruction_overlay,
4249                    "parallelToolCalls": profile.parallel_tool_calls,
4250                    "autoCompactTokenLimit": profile.auto_compact_token_limit,
4251                });
4252            }
4253            let task_ledger_required_this_round = req.task_ledger_required
4254                && runtime_profile == RuntimeProfile::Eval
4255                && !deadline_finalization_requested
4256                && !transcript_has_task_ledger(&transcript);
4257            let task_ledger_tools = (capabilities.tool_calls && task_ledger_required_this_round)
4258                .then(|| self.task_ledger_tool_specs(model_profile.as_ref()))
4259                .filter(|tools| !tools.is_empty());
4260            let request_tools = if deadline_finalization_requested {
4261                Vec::new()
4262            } else if let Some(ledger_tools) = &task_ledger_tools {
4263                ledger_tools.clone()
4264            } else {
4265                tools.clone()
4266            };
4267            let request_tool_choice = if deadline_finalization_requested {
4268                ToolChoice::None
4269            } else if task_ledger_tools.is_some() {
4270                ToolChoice::Specific(TASK_LEDGER_TOOL_NAME.to_string())
4271            } else {
4272                tool_choice.clone()
4273            };
4274            if deadline_finalization_requested {
4275                request_metadata["deadlineFinalization"] = serde_json::json!({
4276                    "reserveSeconds": deadline_finalization_reserve,
4277                    "remainingSeconds": deadline_remaining_seconds(turn_deadline),
4278                });
4279            } else if deadline_scoreable_completion_requested {
4280                request_metadata["deadlineScoreableCompletion"] = serde_json::json!({
4281                    "reserveSeconds": deadline_finalization_reserve,
4282                    "remainingSeconds": deadline_remaining_seconds(turn_deadline),
4283                });
4284            }
4285            let mut eager_tools = EagerTools::new(
4286                self,
4287                &request_tools,
4288                &thread_overrides.external_tools,
4289                parallel_tool_calls,
4290            );
4291            let mut request_reliability: ReliabilityRequestPolicy = cfg.reliability.clone().into();
4292            request_reliability.provider_retry_max_attempts = request_reliability
4293                .provider_retry_max_attempts
4294                .saturating_sub(provider_stream_retry_attempts)
4295                .max(1);
4296            let request = AgentInferenceRequest {
4297                model: ModelSelection {
4298                    provider: provider.clone(),
4299                    model: model.clone(),
4300                },
4301                instructions,
4302                transcript: transcript.clone(),
4303                tools: request_tools,
4304                tool_choice: request_tool_choice,
4305                reasoning: request_reasoning,
4306                output: OutputConfig::default(),
4307                runtime: RuntimeHints {
4308                    auto_compact_token_limit: (provider != roder_api::catalog::PROVIDER_CODEX)
4309                        .then(|| server_side_compaction_threshold(&cfg, &model))
4310                        .flatten(),
4311                    profile: runtime_profile,
4312                    parallel_tool_calls: Some(parallel_tool_calls),
4313                    prompt_cache_key: Some(req.thread_id.clone()),
4314                    hosted_web_search: cfg.hosted_web_search.clone(),
4315                    tool_search: tool_search_for_provider_model(&cfg, &provider, &model),
4316                    speed_policy: speed_policy_decision,
4317                    reliability: Some(request_reliability),
4318                    deadline_remaining_seconds: deadline_remaining_seconds(turn_deadline),
4319                    service_tier: req.service_tier_override.clone(),
4320                    ..RuntimeHints::default()
4321                },
4322                metadata: request_metadata,
4323            };
4324
4325            let mut compaction_template = request.clone();
4326            compaction_template.transcript.clear();
4327            self.compaction_templates
4328                .write()
4329                .await
4330                .insert(req.thread_id.clone(), compaction_template);
4331
4332            let ctx = InferenceTurnContext {
4333                thread_id: &req.thread_id,
4334                turn_id: &turn_id,
4335                tool_executor: Some(std::sync::Arc::new(
4336                    crate::tool_execution::RuntimeTurnToolExecutor {
4337                        runtime: Arc::clone(self),
4338                        thread_id: req.thread_id.clone(),
4339                        turn_id: turn_id.clone(),
4340                        workspace: Some(workspace.clone()),
4341                        deadline: turn_deadline,
4342                    },
4343                )),
4344            };
4345            let stream_future = preemptible(engine.stream_turn(ctx, request), &mut steering);
4346            let mut stream = if let Some((deadline, timeout_action)) = inference_timeout_deadline(
4347                turn_deadline,
4348                runtime_profile,
4349                req.task_ledger_required,
4350                deadline_finalization_reserve,
4351                deadline_finalization_requested || deadline_scoreable_completion_requested,
4352                task_ledger_scoreable_checkpoints,
4353                &transcript,
4354            ) {
4355                match tokio::time::timeout_at(deadline_instant(deadline), stream_future).await {
4356                    Ok(stream) => match stream {
4357                        Ok(stream) => stream,
4358                        Err(err) => {
4359                            if err.is::<SamplingPreempted>() {
4360                                self.finish_sampling_cleanup(&turn_id).await?;
4361                                provider_stream_retry_attempts = 0;
4362                                continue 'tool_rounds;
4363                            }
4364                            self.finish_sampling_cleanup(&turn_id).await?;
4365                            if !deadline_finalization_requested
4366                                && self
4367                                    .retry_sampling_failure(
4368                                        SamplingRetry {
4369                                            thread_id: &req.thread_id,
4370                                            turn_id: &turn_id,
4371                                            provider: &provider,
4372                                            model: &model,
4373                                            config: &cfg.reliability,
4374                                            error: &err,
4375                                        },
4376                                        &mut provider_stream_retry_attempts,
4377                                        &mut steering,
4378                                    )
4379                                    .await
4380                            {
4381                                continue 'tool_rounds;
4382                            }
4383                            let error = err.to_string();
4384                            if self
4385                                .try_recover_from_context_limit(
4386                                    &req.thread_id,
4387                                    &turn_id,
4388                                    &provider,
4389                                    &model,
4390                                    &error,
4391                                    &mut transcript,
4392                                    &mut compacted_this_turn,
4393                                    &mut context_limit_recovery_attempts,
4394                                )
4395                                .await?
4396                            {
4397                                continue 'tool_rounds;
4398                            }
4399                            return Err(err);
4400                        }
4401                    },
4402                    Err(_) => {
4403                        if runtime_profile == RuntimeProfile::Eval
4404                            && !deadline_finalization_requested
4405                        {
4406                            let remaining = deadline_remaining_seconds(turn_deadline).unwrap_or(0);
4407                            if timeout_action == InferenceTimeoutAction::ScoreableCheckpoint
4408                                && task_ledger_scoreable_checkpoints
4409                                    < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
4410                                && let Some(prompt) = task_ledger_completion_prompt(&transcript)
4411                            {
4412                                task_ledger_scoreable_checkpoints += 1;
4413                                let item = TranscriptItem::UserMessage(UserMessage::text(
4414                                    task_ledger_scoreable_checkpoint_prompt(remaining, &prompt),
4415                                ));
4416                                self.persist_turn_item(&req.thread_id, &turn_id, &item)
4417                                    .await?;
4418                                transcript.push(item);
4419                                continue 'tool_rounds;
4420                            }
4421                            self.start_deadline_finalization(
4422                                &req.thread_id,
4423                                &turn_id,
4424                                &mut transcript,
4425                                remaining,
4426                            )
4427                            .await?;
4428                            deadline_finalization_requested = true;
4429                            continue 'tool_rounds;
4430                        }
4431                        self.fail_turn_due_to_deadline(
4432                            &req.thread_id,
4433                            &turn_id,
4434                            deadline,
4435                            &transcript,
4436                        )
4437                        .await?;
4438                        return Ok(TurnRunOutcome::Stopped);
4439                    }
4440                }
4441            } else {
4442                match stream_future.await {
4443                    Ok(stream) => stream,
4444                    Err(err) => {
4445                        if err.is::<SamplingPreempted>() {
4446                            self.finish_sampling_cleanup(&turn_id).await?;
4447                            provider_stream_retry_attempts = 0;
4448                            continue 'tool_rounds;
4449                        }
4450                        self.finish_sampling_cleanup(&turn_id).await?;
4451                        if !deadline_finalization_requested
4452                            && self
4453                                .retry_sampling_failure(
4454                                    SamplingRetry {
4455                                        thread_id: &req.thread_id,
4456                                        turn_id: &turn_id,
4457                                        provider: &provider,
4458                                        model: &model,
4459                                        config: &cfg.reliability,
4460                                        error: &err,
4461                                    },
4462                                    &mut provider_stream_retry_attempts,
4463                                    &mut steering,
4464                                )
4465                                .await
4466                        {
4467                            continue 'tool_rounds;
4468                        }
4469                        let error = err.to_string();
4470                        if self
4471                            .try_recover_from_context_limit(
4472                                &req.thread_id,
4473                                &turn_id,
4474                                &provider,
4475                                &model,
4476                                &error,
4477                                &mut transcript,
4478                                &mut compacted_this_turn,
4479                                &mut context_limit_recovery_attempts,
4480                            )
4481                            .await?
4482                        {
4483                            continue 'tool_rounds;
4484                        }
4485                        return Err(err);
4486                    }
4487                }
4488            };
4489            let mut sampling_output = SamplingOutput::default();
4490            let output_context = OutputContext {
4491                thread_id: &req.thread_id,
4492                turn_id: &turn_id,
4493                profile: model_profile.as_ref(),
4494                provider: &provider,
4495                model: &model,
4496            };
4497            let mut assistant_text = String::new();
4498            let mut phase_messages = Vec::<AssistantMessage>::new();
4499            let mut reasoning_text = String::new();
4500            let mut replayed_tool_call = false;
4501            let mut tool_calls = Vec::new();
4502            let mut patch_progress = PatchProgressTracker::default();
4503            let mut provider_metadata = None;
4504            let mut provider_requests_continuation = false;
4505            // Captured from mid-stream Compaction(completed) events. Codex/OpenAI
4506            // sometimes abort the SSE stream after emitting the compaction item
4507            // ("error decoding response body") before response.completed, so we
4508            // must not rely solely on ProviderMetadata for the boundary.
4509            let mut stream_compaction_item: Option<serde_json::Value> = None;
4510
4511            loop {
4512                let next_future = async {
4513                    loop {
4514                        tokio::select! {
4515                            biased;
4516                            _ = wait_for_steer(&mut steering) => break Some(Err(SamplingPreempted.into())),
4517                            result = eager_tools.poll_result(), if eager_tools.pending() => {
4518                                if let Err(error) = result { break Some(Err(error)); }
4519                            }
4520                            next = stream.next() => break next,
4521                        }
4522                    }
4523                };
4524                let next = if let Some((deadline, timeout_action)) = inference_timeout_deadline(
4525                    turn_deadline,
4526                    runtime_profile,
4527                    req.task_ledger_required,
4528                    deadline_finalization_reserve,
4529                    deadline_finalization_requested || deadline_scoreable_completion_requested,
4530                    task_ledger_scoreable_checkpoints,
4531                    &transcript,
4532                ) {
4533                    match tokio::time::timeout_at(deadline_instant(deadline), next_future).await {
4534                        Ok(next) => next,
4535                        Err(_) => {
4536                            if runtime_profile == RuntimeProfile::Eval
4537                                && !deadline_finalization_requested
4538                            {
4539                                let remaining =
4540                                    deadline_remaining_seconds(turn_deadline).unwrap_or(0);
4541                                if timeout_action == InferenceTimeoutAction::ScoreableCheckpoint
4542                                    && task_ledger_scoreable_checkpoints
4543                                        < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
4544                                    && let Some(prompt) = task_ledger_completion_prompt(&transcript)
4545                                {
4546                                    task_ledger_scoreable_checkpoints += 1;
4547                                    let item = TranscriptItem::UserMessage(UserMessage::text(
4548                                        task_ledger_scoreable_checkpoint_prompt(remaining, &prompt),
4549                                    ));
4550                                    self.persist_turn_item(&req.thread_id, &turn_id, &item)
4551                                        .await?;
4552                                    transcript.push(item);
4553                                    continue 'tool_rounds;
4554                                }
4555                                self.start_deadline_finalization(
4556                                    &req.thread_id,
4557                                    &turn_id,
4558                                    &mut transcript,
4559                                    remaining,
4560                                )
4561                                .await?;
4562                                deadline_finalization_requested = true;
4563                                continue 'tool_rounds;
4564                            }
4565                            self.fail_turn_due_to_deadline(
4566                                &req.thread_id,
4567                                &turn_id,
4568                                deadline,
4569                                &transcript,
4570                            )
4571                            .await?;
4572                            return Ok(TurnRunOutcome::Stopped);
4573                        }
4574                    }
4575                } else {
4576                    next_future.await
4577                };
4578                let res = next.unwrap_or_else(|| {
4579                    Err(roder_api::provider_error::ProviderFailure::new(
4580                        roder_api::provider_error::ProviderFailureKind::StreamInterrupted,
4581                        "provider stream ended before terminal completion",
4582                    )
4583                    .into())
4584                });
4585                let event = match res {
4586                    Ok(event) => event,
4587                    Err(err) => {
4588                        drop(stream);
4589                        eager_tools.drain().await?;
4590                        for result in eager_tools.take_results(&tool_calls) {
4591                            verification_gate.record_tool_result(&result);
4592                            transcript.push(TranscriptItem::ToolResult(result));
4593                        }
4594                        self.finish_sampling_cleanup(&turn_id).await?;
4595                        if err.is::<SamplingPreempted>() {
4596                            provider_stream_retry_attempts = 0;
4597                            continue 'tool_rounds;
4598                        }
4599                        if !deadline_finalization_requested
4600                            && self
4601                                .retry_sampling_failure(
4602                                    SamplingRetry {
4603                                        thread_id: &req.thread_id,
4604                                        turn_id: &turn_id,
4605                                        provider: &provider,
4606                                        model: &model,
4607                                        config: &cfg.reliability,
4608                                        error: &err,
4609                                    },
4610                                    &mut provider_stream_retry_attempts,
4611                                    &mut steering,
4612                                )
4613                                .await
4614                        {
4615                            continue 'tool_rounds;
4616                        }
4617                        let error = err.to_string();
4618                        if self
4619                            .try_recover_from_context_limit(
4620                                &req.thread_id,
4621                                &turn_id,
4622                                &provider,
4623                                &model,
4624                                &error,
4625                                &mut transcript,
4626                                &mut compacted_this_turn,
4627                                &mut context_limit_recovery_attempts,
4628                            )
4629                            .await?
4630                        {
4631                            continue 'tool_rounds;
4632                        }
4633                        return Err(err);
4634                    }
4635                };
4636
4637                let inference_timestamp = OffsetDateTime::now_utc();
4638                self.emit(RoderEvent::InferenceEventReceived(InferenceEventReceived {
4639                    thread_id: req.thread_id.clone(),
4640                    turn_id: turn_id.clone(),
4641                    event: event.clone(),
4642                    timestamp: inference_timestamp,
4643                }))
4644                .await;
4645
4646                match event {
4647                    InferenceEvent::MessageDelta(delta) => {
4648                        if let Some((team_id, member)) =
4649                            self.teams.member_for_thread(&req.thread_id).await
4650                        {
4651                            self.emit(RoderEvent::TeamMemberMessageDelta(TeamMemberMessageDelta {
4652                                team_id,
4653                                member_id: member.id,
4654                                member_thread_id: req.thread_id.clone(),
4655                                turn_id: turn_id.clone(),
4656                                delta: delta.text.clone(),
4657                                timestamp: OffsetDateTime::now_utc(),
4658                            }))
4659                            .await;
4660                        }
4661                        if is_final_answer_phase(delta.phase.as_deref()) {
4662                            assistant_text.push_str(&delta.text);
4663                        } else if let Some(last) = phase_messages.last_mut()
4664                            && last.phase == delta.phase
4665                        {
4666                            last.text.push_str(&delta.text);
4667                        } else {
4668                            phase_messages.push(AssistantMessage {
4669                                text: delta.text,
4670                                phase: delta.phase,
4671                            });
4672                        }
4673                    }
4674                    InferenceEvent::ReasoningDelta(delta) => reasoning_text.push_str(&delta.text),
4675                    InferenceEvent::ToolCallCompleted(call) => {
4676                        if completed_call_replayed(&transcript, &call)? {
4677                            replayed_tool_call = true;
4678                            continue;
4679                        }
4680                        if let Some(progress) = patch_progress.complete(
4681                            &req.thread_id,
4682                            &turn_id,
4683                            &call.id,
4684                            &call.name,
4685                            &call.arguments,
4686                        ) {
4687                            self.emit(RoderEvent::PatchProgress(progress)).await;
4688                        }
4689                        if !tool_calls
4690                            .iter()
4691                            .any(|previous: &ToolCallCompleted| previous.id == call.id)
4692                        {
4693                            eager_tools
4694                                .schedule(
4695                                    self,
4696                                    &output_context,
4697                                    &call,
4698                                    workspace.as_str(),
4699                                    turn_deadline,
4700                                    &mut transcript,
4701                                )
4702                                .await?;
4703                            tool_calls.push(call);
4704                        }
4705                    }
4706                    InferenceEvent::OutputItemCompleted(item) => {
4707                        sampling_output
4708                            .complete_item(self, &output_context, item, &mut transcript)
4709                            .await?;
4710                    }
4711                    InferenceEvent::Failed(failure) => {
4712                        speed_policy.record_failure();
4713                        let error = failure.message;
4714                        if self
4715                            .try_recover_from_context_limit(
4716                                &req.thread_id,
4717                                &turn_id,
4718                                &provider,
4719                                &model,
4720                                &error,
4721                                &mut transcript,
4722                                &mut compacted_this_turn,
4723                                &mut context_limit_recovery_attempts,
4724                            )
4725                            .await?
4726                        {
4727                            continue 'tool_rounds;
4728                        }
4729                        self.persist_turn_item(
4730                            &req.thread_id,
4731                            &turn_id,
4732                            &TranscriptItem::Error(ErrorRecord {
4733                                message: error.clone(),
4734                            }),
4735                        )
4736                        .await?;
4737                        self.emit(RoderEvent::TurnFailed(TurnFailed {
4738                            thread_id: req.thread_id.clone(),
4739                            turn_id: turn_id.clone(),
4740                            error: error.clone(),
4741                            error_kind: None,
4742                            usage: None,
4743                            timestamp: OffsetDateTime::now_utc(),
4744                        }))
4745                        .await;
4746                        self.complete_team_member_turn_with_result(
4747                            &req.thread_id,
4748                            &turn_id,
4749                            TeamMemberStatus::Failed,
4750                            (!assistant_text.trim().is_empty()).then(|| assistant_text.clone()),
4751                            Some(error),
4752                        )
4753                        .await?;
4754                        return Ok(TurnRunOutcome::Stopped);
4755                    }
4756                    InferenceEvent::Usage(usage) => {
4757                        turn_usage.add_assign(&usage);
4758                    }
4759                    InferenceEvent::Completed(metadata) => {
4760                        sampling_output
4761                            .finish(
4762                                self,
4763                                &output_context,
4764                                &assistant_text,
4765                                &phase_messages,
4766                                &reasoning_text,
4767                                &mut transcript,
4768                            )
4769                            .await?;
4770                        provider_stream_retry_attempts = 0;
4771                        turn_finish_reason = metadata
4772                            .stop_reason
4773                            .as_deref()
4774                            .map(finish_reason_from_stop_reason);
4775                        break;
4776                    }
4777                    InferenceEvent::Compaction(progress) => {
4778                        // Persist the boundary as soon as the provider emits a
4779                        // completed compaction item. ChatGPT Codex often aborts
4780                        // the SSE stream right after ("error decoding response
4781                        // body"), so waiting for ProviderMetadata loses the
4782                        // boundary and every subsequent request re-compacts.
4783                        if progress.status == "completed"
4784                            && let Some(item) = progress.item
4785                        {
4786                            let already = stream_compaction_item
4787                                .as_ref()
4788                                .and_then(|existing| {
4789                                    existing.get("id").and_then(serde_json::Value::as_str)
4790                                })
4791                                .zip(item.get("id").and_then(serde_json::Value::as_str))
4792                                .is_some_and(|(a, b)| a == b);
4793                            if !already {
4794                                stream_compaction_item = Some(item.clone());
4795                                let boundary = TranscriptItem::ProviderMetadata(
4796                                    crate::compaction::provider_metadata_from_compaction_item(item),
4797                                );
4798                                self.persist_turn_item(&req.thread_id, &turn_id, &boundary)
4799                                    .await?;
4800                                transcript.push(boundary);
4801                                transcript =
4802                                    crate::compaction::trim_to_last_compaction_boundary(transcript);
4803                                compacted_this_turn = true;
4804                            }
4805                        }
4806                    }
4807                    InferenceEvent::HostedToolCallStarted(_)
4808                    | InferenceEvent::HostedToolCallCompleted(_) => {}
4809                    InferenceEvent::ToolCallStarted(call) => {
4810                        patch_progress.start(&call.id, &call.name)
4811                    }
4812                    InferenceEvent::ToolCallDelta(delta) => {
4813                        if let Some(progress) = patch_progress.delta(
4814                            &req.thread_id,
4815                            &turn_id,
4816                            &delta.id,
4817                            &delta.arguments_delta,
4818                        ) {
4819                            self.emit(RoderEvent::PatchProgress(progress)).await;
4820                        }
4821                    }
4822                    InferenceEvent::ProviderMetadata(mut metadata) => {
4823                        if metadata.get("kind").and_then(serde_json::Value::as_str)
4824                            == Some("reliability_retry_attempt")
4825                        {
4826                            provider_stream_retry_attempts =
4827                                provider_stream_retry_attempts.saturating_add(1);
4828                        }
4829                        provider_requests_continuation |= metadata
4830                            .get("end_turn")
4831                            .and_then(serde_json::Value::as_bool)
4832                            == Some(false);
4833                        if let Some(compaction_item) = stream_compaction_item.as_ref() {
4834                            let _ = crate::compaction::ensure_provider_metadata_compaction(
4835                                &mut metadata,
4836                                compaction_item,
4837                            );
4838                        }
4839                        if metadata.get("output").is_none() {
4840                            let item = TranscriptItem::ProviderMetadata(metadata);
4841                            self.persist_turn_item(&req.thread_id, &turn_id, &item)
4842                                .await?;
4843                            transcript.push(item);
4844                        } else {
4845                            provider_metadata = Some(metadata);
4846                        }
4847                    }
4848                }
4849            }
4850
4851            speed_policy.record_model_output(
4852                !assistant_text.is_empty() || !phase_messages.is_empty(),
4853                tool_calls.len(),
4854            );
4855            if tool_calls.is_empty() {
4856                if provider_requests_continuation || replayed_tool_call {
4857                    if let Some(metadata) = provider_metadata {
4858                        let item = TranscriptItem::ProviderMetadata(metadata);
4859                        self.persist_turn_item(&req.thread_id, &turn_id, &item)
4860                            .await?;
4861                        transcript.push(item);
4862                    }
4863                    continue 'tool_rounds;
4864                }
4865                let steers = self.drain_turn_steers(&turn_id, &mut steering).await;
4866                if !steers.is_empty() {
4867                    if let Some(metadata) = provider_metadata {
4868                        let had_provider_compaction =
4869                            crate::compaction::provider_metadata_has_compaction(&metadata);
4870                        let item = TranscriptItem::ProviderMetadata(metadata);
4871                        self.persist_turn_item(&req.thread_id, &turn_id, &item)
4872                            .await?;
4873                        transcript.push(item);
4874                        if had_provider_compaction {
4875                            // Server-side compaction replaced the prior window;
4876                            // drop pre-boundary items so the next request does
4877                            // not re-send (and re-compact) the full history.
4878                            transcript =
4879                                crate::compaction::trim_to_last_compaction_boundary(transcript);
4880                            compacted_this_turn = true;
4881                        }
4882                    }
4883                    self.append_steers(&req, &turn_id, &mut transcript, steers)
4884                        .await?;
4885                    continue;
4886                }
4887                if !deadline_finalization_requested
4888                    && req.task_ledger_required
4889                    && runtime_profile == RuntimeProfile::Eval
4890                    && task_ledger_completion_reminders < TASK_LEDGER_COMPLETION_REMINDER_LIMIT
4891                    && (!assistant_text.trim().is_empty() || !phase_messages.is_empty())
4892                    && let Some(prompt) = task_ledger_completion_prompt(&transcript)
4893                {
4894                    task_ledger_completion_reminders += 1;
4895                    let item = TranscriptItem::UserMessage(UserMessage::text(prompt));
4896                    self.persist_turn_item(&req.thread_id, &turn_id, &item)
4897                        .await?;
4898                    transcript.push(item);
4899                    continue;
4900                }
4901                if !deadline_finalization_requested
4902                    && let Some(prompt) = verification_gate.blocking_prompt()
4903                {
4904                    speed_policy.record_verification_required();
4905                    self.emit(RoderEvent::VerificationRequired(VerificationRequired {
4906                        thread_id: req.thread_id.clone(),
4907                        turn_id: turn_id.clone(),
4908                        reason: verification_gate.reason(),
4909                        changed_files: verification_gate.changed_files(),
4910                        tool_evidence: verification_gate.tool_evidence.clone(),
4911                        tests_run: verification_gate.tests_run.clone(),
4912                        open_gaps: verification_gate.open_gaps.clone(),
4913                        timestamp: OffsetDateTime::now_utc(),
4914                    }))
4915                    .await;
4916                    let item = TranscriptItem::UserMessage(UserMessage::text(prompt));
4917                    self.persist_turn_item(&req.thread_id, &turn_id, &item)
4918                        .await?;
4919                    transcript.push(item);
4920                    continue;
4921                }
4922                // Loop persistence nudge: in non-interactive/eval runs, when the
4923                // model returns a final message with no tool calls, gently prompt
4924                // it to keep going (bounded by `empty_tool_call_nudges`) before
4925                // ending the turn. Gated to non-interactive/eval so interactive
4926                // chat completions are never delayed, and only fires when the
4927                // model actually produced a final answer to re-examine.
4928                if !deadline_finalization_requested
4929                    && runtime_profile != RuntimeProfile::Interactive
4930                    && cfg.reliability.empty_tool_call_nudges > 0
4931                    && empty_tool_call_nudges_used < cfg.reliability.empty_tool_call_nudges
4932                    && (!assistant_text.trim().is_empty() || !phase_messages.is_empty())
4933                {
4934                    empty_tool_call_nudges_used += 1;
4935                    let item = TranscriptItem::UserMessage(UserMessage::text(
4936                        RELIABILITY_CONTINUATION_PROMPT.to_string(),
4937                    ));
4938                    self.persist_turn_item(&req.thread_id, &turn_id, &item)
4939                        .await?;
4940                    transcript.push(item);
4941                    continue;
4942                }
4943                if deadline_finalization_requested
4944                    && assistant_text.trim().is_empty()
4945                    && phase_messages.is_empty()
4946                {
4947                    assistant_text = format!(
4948                        "Deadline finalization completed without model text. {}",
4949                        turn_partial_result(&transcript)
4950                    );
4951                }
4952                final_assistant_text = assistant_text;
4953                final_provider_metadata = provider_metadata;
4954                exhausted_tool_rounds = false;
4955                break;
4956            }
4957
4958            if let Some(metadata) = provider_metadata {
4959                let had_provider_compaction =
4960                    crate::compaction::provider_metadata_has_compaction(&metadata);
4961                let item = TranscriptItem::ProviderMetadata(metadata);
4962                self.persist_turn_item(&req.thread_id, &turn_id, &item)
4963                    .await?;
4964                transcript.push(item);
4965                if had_provider_compaction {
4966                    // Server-side compaction replaced the prior window; drop
4967                    // pre-boundary items so subsequent tool rounds use the
4968                    // compact window as the next input.
4969                    transcript = crate::compaction::trim_to_last_compaction_boundary(transcript);
4970                    compacted_this_turn = true;
4971                }
4972            }
4973            eager_tools.drain().await?;
4974            for call in &tool_calls {
4975                if eager_tools.scheduled(&call.id) {
4976                    continue;
4977                }
4978                let tool_item = TranscriptItem::ToolCall(ToolCallRecord {
4979                    id: call.id.clone(),
4980                    name: call.name.clone(),
4981                    arguments: call.arguments.clone(),
4982                });
4983                self.persist_turn_item(&req.thread_id, &turn_id, &tool_item)
4984                    .await?;
4985                transcript.push(tool_item);
4986                self.persist_model_profile_segment(
4987                    &req.thread_id,
4988                    &turn_id,
4989                    model_profile.as_ref(),
4990                    &provider,
4991                    &model,
4992                    "tool_call",
4993                )
4994                .await?;
4995            }
4996            if let Some(deadline) = turn_deadline
4997                && deadline_expired(deadline)
4998            {
4999                self.fail_turn_due_to_deadline(&req.thread_id, &turn_id, deadline, &transcript)
5000                    .await?;
5001                return Ok(TurnRunOutcome::Stopped);
5002            }
5003            let mut results = eager_tools.take_results(&tool_calls);
5004            let remaining_calls = tool_calls
5005                .into_iter()
5006                .filter(|call| !eager_tools.scheduled(&call.id))
5007                .collect();
5008            results.extend(
5009                self.route_tool_calls(
5010                    &req.thread_id,
5011                    &turn_id,
5012                    remaining_calls,
5013                    parallel_tool_calls,
5014                    Some(workspace.as_str()),
5015                    turn_deadline,
5016                )
5017                .await?,
5018            );
5019            let reliability_limit = reliability.record_tool_results(
5020                &cfg.reliability,
5021                &results,
5022                runtime_profile == RuntimeProfile::Interactive,
5023            );
5024            for result in results {
5025                verification_gate.record_tool_result(&result);
5026                transcript.push(TranscriptItem::ToolResult(result));
5027                self.persist_model_profile_segment(
5028                    &req.thread_id,
5029                    &turn_id,
5030                    model_profile.as_ref(),
5031                    &provider,
5032                    &model,
5033                    "tool_result",
5034                )
5035                .await?;
5036            }
5037            if let Some(limit) = reliability_limit {
5038                if limit.decision == ReliabilityLimitDecision::RequestContinuation {
5039                    // Loop persistence: rather than ending the turn, reset the
5040                    // consecutive-failure counter, nudge the model to keep going,
5041                    // and continue the round loop. Bounded by the per-turn tool
5042                    // failure ceiling, `max_model_calls_per_turn`, and
5043                    // `MAX_TOOL_ROUNDS_PER_TURN`.
5044                    self.record_reliability_limit_continuation(
5045                        &req.thread_id,
5046                        &turn_id,
5047                        &provider,
5048                        &model,
5049                        limit,
5050                    )
5051                    .await;
5052                    reliability.reset_consecutive_failures();
5053                    let nudge = TranscriptItem::UserMessage(UserMessage::text(
5054                        RELIABILITY_CONTINUATION_PROMPT.to_string(),
5055                    ));
5056                    self.persist_turn_item(&req.thread_id, &turn_id, &nudge)
5057                        .await?;
5058                    transcript.push(nudge);
5059                } else {
5060                    self.fail_turn_due_to_reliability_limit(
5061                        &req.thread_id,
5062                        &turn_id,
5063                        &provider,
5064                        &model,
5065                        limit,
5066                        &transcript,
5067                    )
5068                    .await?;
5069                    return Ok(TurnRunOutcome::Stopped);
5070                }
5071            }
5072            let compaction_generation_before = self.compaction_generation(&req.thread_id);
5073            transcript = self
5074                .compact_transcript_if_needed(
5075                    &req.thread_id,
5076                    &turn_id,
5077                    &provider,
5078                    &model,
5079                    transcript,
5080                    self.compaction_options_for_turn(&req.thread_id, !compacted_this_turn),
5081                )
5082                .await?;
5083            compacted_this_turn |=
5084                self.compaction_generation(&req.thread_id) != compaction_generation_before;
5085        }
5086
5087        if exhausted_tool_rounds {
5088            let message =
5089                format!("tool call limit reached after {MAX_TOOL_ROUNDS_PER_TURN} rounds");
5090            self.persist_turn_item(
5091                &req.thread_id,
5092                &turn_id,
5093                &TranscriptItem::Error(ErrorRecord {
5094                    message: message.clone(),
5095                }),
5096            )
5097            .await?;
5098            self.emit(RoderEvent::TurnFailed(TurnFailed {
5099                thread_id: req.thread_id.clone(),
5100                turn_id: turn_id.clone(),
5101                error: message.clone(),
5102                error_kind: None,
5103                usage: None,
5104                timestamp: OffsetDateTime::now_utc(),
5105            }))
5106            .await;
5107            self.complete_team_member_turn_with_result(
5108                &req.thread_id,
5109                &turn_id,
5110                TeamMemberStatus::Failed,
5111                None,
5112                Some(message),
5113            )
5114            .await?;
5115            return Ok(TurnRunOutcome::Stopped);
5116        }
5117
5118        if let Some(metadata) = final_provider_metadata {
5119            self.persist_turn_item(
5120                &req.thread_id,
5121                &turn_id,
5122                &TranscriptItem::ProviderMetadata(metadata),
5123            )
5124            .await?;
5125        }
5126
5127        let turn_usage_tokens = turn_usage.total_tokens as i64;
5128        let completed_usage = (!turn_usage.is_empty()).then_some(turn_usage.clone());
5129        self.record_thread_usage_metadata(&req.thread_id, &turn_usage)
5130            .await?;
5131        self.goals
5132            .account_turn_usage(
5133                &req.thread_id,
5134                turn_usage_tokens,
5135                OffsetDateTime::now_utc() - turn_started_at,
5136            )
5137            .await?;
5138        let (cleanup, ownership) = self.await_provider_turn_cleanup(&turn_id).await;
5139        self.record_turn_lifecycle_with_ownership(
5140            req.thread_id.clone(),
5141            turn_id.clone(),
5142            TurnLifecycleState::Completed,
5143            cleanup,
5144            None,
5145            ownership,
5146        )
5147        .await;
5148        self.emit(RoderEvent::TurnCompleted(TurnCompleted {
5149            thread_id: req.thread_id.clone(),
5150            turn_id: turn_id.clone(),
5151            usage: completed_usage,
5152            finish_reason: turn_finish_reason,
5153            timestamp: OffsetDateTime::now_utc(),
5154        }))
5155        .await;
5156        self.complete_team_member_turn_with_result(
5157            &req.thread_id,
5158            &turn_id,
5159            TeamMemberStatus::Completed,
5160            (!final_assistant_text.is_empty()).then_some(final_assistant_text),
5161            None,
5162        )
5163        .await?;
5164        Ok(TurnRunOutcome::Completed)
5165    }
5166
5167    async fn route_tool_calls(
5168        self: &Arc<Self>,
5169        thread_id: &ThreadId,
5170        turn_id: &TurnId,
5171        calls: Vec<ToolCallCompleted>,
5172        parallel: bool,
5173        workspace: Option<&str>,
5174        deadline: Option<OffsetDateTime>,
5175    ) -> anyhow::Result<Vec<ToolResultRecord>> {
5176        // Enforce the agent_swarm exclusivity rule (roadmap 104, Task 2):
5177        // `agent_swarm` must be the only tool call in a model response. A mixed
5178        // or multi-swarm batch is denied wholesale with actionable retry text so
5179        // the model re-issues `agent_swarm` by itself, and every tool_call_id
5180        // still gets a response (keeping the chat-completions transcript valid).
5181        if let Some(violation) = roder_api::subagents::agent_swarm_batch_violation(
5182            calls.iter().map(|call| call.name.as_str()),
5183        ) {
5184            let message = violation.deny_message();
5185            return Ok(calls
5186                .into_iter()
5187                .map(|call| ToolResultRecord {
5188                    id: call.id,
5189                    name: Some(call.name),
5190                    result: message.clone(),
5191                    display_payload: None,
5192                    is_error: true,
5193                })
5194                .collect());
5195        }
5196        // Emit swarm lifecycle events on the bus (roadmap 104, Task 1) so any
5197        // app-server/SDK/TUI client can observe a swarm as a whole; per-child
5198        // progress flows through the existing Subagent* trace events.
5199        let swarm_call = (calls.len() == 1
5200            && calls[0].name == roder_api::subagents::AGENT_SWARM_TOOL_NAME)
5201            .then(|| (calls[0].id.clone(), calls[0].arguments.clone()));
5202        if let Some((tool_id, args)) = &swarm_call {
5203            self.emit(RoderEvent::AgentSwarmStarted(
5204                roder_api::subagents::AgentSwarmStarted {
5205                    thread_id: thread_id.clone(),
5206                    turn_id: turn_id.clone(),
5207                    tool_id: tool_id.clone(),
5208                    child_count: agent_swarm_child_count(args),
5209                    timestamp: OffsetDateTime::now_utc(),
5210                },
5211            ))
5212            .await;
5213        }
5214
5215        let force_sequential = calls
5216            .iter()
5217            .any(|call| crate::agent_control_tools::is_agent_control_tool(&call.name));
5218        let results =
5219            if parallel && !force_sequential {
5220                try_join_all(calls.into_iter().map(|call| {
5221                    self.route_tool_call(thread_id, turn_id, call, workspace, deadline)
5222                }))
5223                .await
5224            } else {
5225                let mut results = Vec::with_capacity(calls.len());
5226                for call in calls {
5227                    results.push(
5228                        self.route_tool_call(thread_id, turn_id, call, workspace, deadline)
5229                            .await?,
5230                    );
5231                }
5232                Ok(results)
5233            }?;
5234
5235        if let Some((tool_id, _)) = &swarm_call
5236            && let Some(result) = results.iter().find(|result| &result.id == tool_id)
5237            && let Some((completed, failed, aborted)) = parse_swarm_counts(&result.result)
5238        {
5239            self.emit(RoderEvent::AgentSwarmCompleted(
5240                roder_api::subagents::AgentSwarmCompleted {
5241                    thread_id: thread_id.clone(),
5242                    turn_id: turn_id.clone(),
5243                    tool_id: tool_id.clone(),
5244                    completed,
5245                    failed,
5246                    aborted,
5247                    timestamp: OffsetDateTime::now_utc(),
5248                },
5249            ))
5250            .await;
5251        }
5252
5253        Ok(results)
5254    }
5255
5256    /// On provider context/prompt-length failures, force-compact (and if needed
5257    /// strip the last bulky item) then retry the current tool round instead of
5258    /// failing the turn. Bounded so a permanently oversized suffix still stops.
5259    async fn try_recover_from_context_limit(
5260        &self,
5261        thread_id: &ThreadId,
5262        turn_id: &TurnId,
5263        provider: &str,
5264        model: &str,
5265        error: &str,
5266        transcript: &mut Vec<TranscriptItem>,
5267        compacted_this_turn: &mut bool,
5268        recovery_attempts: &mut u32,
5269    ) -> anyhow::Result<bool> {
5270        const MAX_CONTEXT_LIMIT_RECOVERY_ATTEMPTS: u32 = 2;
5271        if !crate::compaction::is_context_limit_failure_message(error) {
5272            return Ok(false);
5273        }
5274        if *recovery_attempts >= MAX_CONTEXT_LIMIT_RECOVERY_ATTEMPTS {
5275            return Ok(false);
5276        }
5277        *recovery_attempts = recovery_attempts.saturating_add(1);
5278
5279        let error_item = TranscriptItem::Error(ErrorRecord {
5280            message: error.to_string(),
5281        });
5282        self.persist_turn_item(thread_id, turn_id, &error_item)
5283            .await?;
5284        transcript.push(error_item);
5285
5286        // After the first failed recovery, drop the item ahead of the latest
5287        // user/error so a second compact can succeed when the suffix itself is
5288        // still oversized (e.g. a huge tool result right before the prompt).
5289        if *recovery_attempts > 1 {
5290            *transcript =
5291                crate::compaction::strip_last_message_before_error(std::mem::take(transcript));
5292        }
5293
5294        let force_options = crate::compaction::CompactionOptions {
5295            allow_repeat: true,
5296            force: true,
5297            hysteresis_baseline: None,
5298            preserve_hint: Some(
5299                "Recover from a provider context/prompt-length limit; preserve the active user goal and recent tool outcomes."
5300                    .to_string(),
5301            ),
5302        };
5303        let before_len = transcript.len();
5304        let before_tokens = crate::compaction::estimate_prompt_tokens(transcript);
5305        *transcript = self
5306            .compact_transcript_if_needed(
5307                thread_id,
5308                turn_id,
5309                provider,
5310                model,
5311                std::mem::take(transcript),
5312                force_options,
5313            )
5314            .await?;
5315        let after_tokens = crate::compaction::estimate_prompt_tokens(transcript);
5316        *compacted_this_turn = *compacted_this_turn
5317            || transcript
5318                .iter()
5319                .any(crate::compaction::is_compaction_boundary);
5320
5321        self.emit(RoderEvent::ReliabilityRetryRecorded(
5322            ReliabilityRetryRecorded {
5323                context: ReliabilityContext {
5324                    thread_id: thread_id.clone(),
5325                    turn_id: turn_id.clone(),
5326                    provider: Some(provider.to_string()),
5327                    model: Some(model.to_string()),
5328                    ..ReliabilityContext::default()
5329                },
5330                error_class: ReliabilityErrorClass::ProviderError,
5331                decision: ReliabilityRetryDecision::Retry,
5332                attempt: *recovery_attempts,
5333                max_attempts: MAX_CONTEXT_LIMIT_RECOVERY_ATTEMPTS,
5334                delay_ms: Some(0),
5335                details: ReliabilityDetails::redacted(format!(
5336                    "context_limit_recovery: compact {before_tokens}->{after_tokens} tokens (items {before_len}->{}); {error}",
5337                    transcript.len()
5338                )),
5339                timestamp: OffsetDateTime::now_utc(),
5340            },
5341        ))
5342        .await;
5343
5344        // If force-compact did not shrink anything and we still have budget,
5345        // strip once more so the next round is not identical.
5346        if after_tokens >= before_tokens && *recovery_attempts < MAX_CONTEXT_LIMIT_RECOVERY_ATTEMPTS
5347        {
5348            *transcript =
5349                crate::compaction::strip_last_message_before_error(std::mem::take(transcript));
5350        }
5351        Ok(true)
5352    }
5353
5354    async fn fail_turn_with_error(
5355        &self,
5356        thread_id: &ThreadId,
5357        turn_id: &TurnId,
5358        message: String,
5359    ) -> anyhow::Result<()> {
5360        self.persist_turn_item(
5361            thread_id,
5362            turn_id,
5363            &TranscriptItem::Error(ErrorRecord {
5364                message: message.clone(),
5365            }),
5366        )
5367        .await?;
5368        self.emit(RoderEvent::TurnFailed(TurnFailed {
5369            thread_id: thread_id.clone(),
5370            turn_id: turn_id.clone(),
5371            error: message.clone(),
5372            error_kind: None,
5373            usage: None,
5374            timestamp: OffsetDateTime::now_utc(),
5375        }))
5376        .await;
5377        self.complete_team_member_turn_with_result(
5378            thread_id,
5379            turn_id,
5380            TeamMemberStatus::Failed,
5381            None,
5382            Some(message),
5383        )
5384        .await?;
5385        Ok(())
5386    }
5387
5388    async fn fail_turn_due_to_deadline(
5389        &self,
5390        thread_id: &ThreadId,
5391        turn_id: &TurnId,
5392        deadline: OffsetDateTime,
5393        transcript: &[TranscriptItem],
5394    ) -> anyhow::Result<()> {
5395        let partial_result = turn_partial_result(transcript);
5396        self.emit(RoderEvent::TurnPartialResult(TurnPartialResult {
5397            thread_id: thread_id.clone(),
5398            turn_id: turn_id.clone(),
5399            summary: partial_result.clone(),
5400            timestamp: OffsetDateTime::now_utc(),
5401        }))
5402        .await;
5403        self.emit(RoderEvent::TurnDeadlineExceeded(TurnDeadlineExceeded {
5404            thread_id: thread_id.clone(),
5405            turn_id: turn_id.clone(),
5406            deadline,
5407            partial_result: partial_result.clone(),
5408            timestamp: OffsetDateTime::now_utc(),
5409        }))
5410        .await;
5411        let message = "turn deadline expired".to_string();
5412        self.persist_turn_item(
5413            thread_id,
5414            turn_id,
5415            &TranscriptItem::Error(ErrorRecord {
5416                message: format!("{message}: {partial_result}"),
5417            }),
5418        )
5419        .await?;
5420        self.emit(RoderEvent::TurnFailed(TurnFailed {
5421            thread_id: thread_id.clone(),
5422            turn_id: turn_id.clone(),
5423            error: message.clone(),
5424            error_kind: Some("deadline_timeout".to_string()),
5425            usage: None,
5426            timestamp: OffsetDateTime::now_utc(),
5427        }))
5428        .await;
5429        self.complete_team_member_turn_with_result(
5430            thread_id,
5431            turn_id,
5432            TeamMemberStatus::Failed,
5433            None,
5434            Some(format!("{message}: {partial_result}")),
5435        )
5436        .await?;
5437        Ok(())
5438    }
5439
5440    async fn start_deadline_finalization(
5441        &self,
5442        thread_id: &ThreadId,
5443        turn_id: &TurnId,
5444        transcript: &mut Vec<TranscriptItem>,
5445        remaining_seconds: u64,
5446    ) -> anyhow::Result<()> {
5447        let item = TranscriptItem::UserMessage(crate::deadline_policy::finalization_message(
5448            remaining_seconds,
5449        ));
5450        self.persist_turn_item(thread_id, turn_id, &item).await?;
5451        transcript.push(item);
5452        self.emit(RoderEvent::TurnPartialResult(TurnPartialResult {
5453            thread_id: thread_id.clone(),
5454            turn_id: turn_id.clone(),
5455            summary: turn_partial_result(transcript),
5456            timestamp: OffsetDateTime::now_utc(),
5457        }))
5458        .await;
5459        Ok(())
5460    }
5461
5462    /// Emits the reliability-limit event for a continuation (loop persistence)
5463    /// without failing the turn. The caller resets the failure counter and
5464    /// injects a nudge so the round loop keeps going.
5465    async fn record_reliability_limit_continuation(
5466        &self,
5467        thread_id: &ThreadId,
5468        turn_id: &TurnId,
5469        provider: &str,
5470        model: &str,
5471        limit: ReliabilityLimitHit,
5472    ) {
5473        self.emit(RoderEvent::ReliabilityLimitRecorded(
5474            ReliabilityLimitRecorded {
5475                context: ReliabilityContext {
5476                    thread_id: thread_id.clone(),
5477                    turn_id: turn_id.clone(),
5478                    tool_id: None,
5479                    tool_name: None,
5480                    provider: Some(provider.to_string()),
5481                    model: Some(model.to_string()),
5482                },
5483                error_class: limit.error_class,
5484                limit_kind: limit.limit_kind,
5485                decision: limit.decision,
5486                current: limit.current,
5487                limit: limit.limit,
5488                details: ReliabilityDetails::redacted(&limit.message),
5489                timestamp: OffsetDateTime::now_utc(),
5490            },
5491        ))
5492        .await;
5493    }
5494
5495    async fn fail_turn_due_to_reliability_limit(
5496        &self,
5497        thread_id: &ThreadId,
5498        turn_id: &TurnId,
5499        provider: &str,
5500        model: &str,
5501        limit: ReliabilityLimitHit,
5502        transcript: &[TranscriptItem],
5503    ) -> anyhow::Result<()> {
5504        self.emit(RoderEvent::ReliabilityLimitRecorded(
5505            ReliabilityLimitRecorded {
5506                context: ReliabilityContext {
5507                    thread_id: thread_id.clone(),
5508                    turn_id: turn_id.clone(),
5509                    tool_id: None,
5510                    tool_name: None,
5511                    provider: Some(provider.to_string()),
5512                    model: Some(model.to_string()),
5513                },
5514                error_class: limit.error_class,
5515                limit_kind: limit.limit_kind,
5516                decision: limit.decision,
5517                current: limit.current,
5518                limit: limit.limit,
5519                details: ReliabilityDetails::redacted(&limit.message),
5520                timestamp: OffsetDateTime::now_utc(),
5521            },
5522        ))
5523        .await;
5524        let partial_result = turn_partial_result(transcript);
5525        self.emit(RoderEvent::TurnPartialResult(TurnPartialResult {
5526            thread_id: thread_id.clone(),
5527            turn_id: turn_id.clone(),
5528            summary: partial_result.clone(),
5529            timestamp: OffsetDateTime::now_utc(),
5530        }))
5531        .await;
5532        let message = format!("reliability limit reached: {}", limit.message);
5533        self.persist_turn_item(
5534            thread_id,
5535            turn_id,
5536            &TranscriptItem::Error(ErrorRecord {
5537                message: format!("{message}: {partial_result}"),
5538            }),
5539        )
5540        .await?;
5541        self.emit(RoderEvent::TurnFailed(TurnFailed {
5542            thread_id: thread_id.clone(),
5543            turn_id: turn_id.clone(),
5544            error: message.clone(),
5545            error_kind: Some("reliability_limit".to_string()),
5546            usage: None,
5547            timestamp: OffsetDateTime::now_utc(),
5548        }))
5549        .await;
5550        self.complete_team_member_turn_with_result(
5551            thread_id,
5552            turn_id,
5553            TeamMemberStatus::Failed,
5554            None,
5555            Some(format!("{message}: {partial_result}")),
5556        )
5557        .await?;
5558        Ok(())
5559    }
5560
5561    async fn append_steers(
5562        &self,
5563        req: &StartTurnRequest,
5564        turn_id: &TurnId,
5565        transcript: &mut Vec<TranscriptItem>,
5566        steers: Vec<QueuedTurnSteer>,
5567    ) -> anyhow::Result<()> {
5568        for queued in steers {
5569            let mut steer = queued.message;
5570            steer.text = steer.text.trim().to_string();
5571            if steer.text.is_empty() && steer.images.is_empty() {
5572                continue;
5573            }
5574            let item = TranscriptItem::UserMessage(steer);
5575            self.persist_turn_item(&req.thread_id, turn_id, &item)
5576                .await?;
5577            transcript.push(item);
5578            if let Some(ack) = queued.mailbox_ack {
5579                self.teams
5580                    .mark_mailbox_messages_delivered(&ack.team_id, turn_id, &ack.message_ids)
5581                    .await?;
5582            }
5583        }
5584        Ok(())
5585    }
5586
5587    async fn persist_model_profile_segment(
5588        &self,
5589        thread_id: &ThreadId,
5590        turn_id: &TurnId,
5591        profile: Option<&ModelHarnessProfile>,
5592        provider: &str,
5593        model: &str,
5594        segment: &str,
5595    ) -> anyhow::Result<()> {
5596        let item = TranscriptItem::ProviderMetadata(model_profile_segment_metadata(
5597            profile, provider, model, segment,
5598        ));
5599        self.persist_turn_item(thread_id, turn_id, &item).await
5600    }
5601
5602    /**
5603     * Allowlists apply to built-in tools only; external tools are advertised with their
5604     * host-supplied schemas as given. An external tool shadows a built-in with the same name in
5605     * both advertisement and dispatch (see `route_tool_call`).
5606     */
5607    fn filtered_tool_specs(
5608        &self,
5609        cfg: &RuntimeConfig,
5610        model: &str,
5611        profile: Option<&ModelHarnessProfile>,
5612        thread_allowlist: &[String],
5613        external_tools: &[roder_api::tools::ToolSpec],
5614    ) -> Vec<roder_api::tools::ToolSpec> {
5615        let mut specs = self
5616            .tool_registry
5617            .specs_for_edit_tool_with_schema_policy(
5618                edit_tool_for_model(cfg, model),
5619                schema_policy_for_model(profile),
5620            )
5621            .into_iter()
5622            .filter(|spec| {
5623                allowlist_permits(&cfg.tool_allowlist, &spec.name)
5624                    && allowlist_permits(thread_allowlist, &spec.name)
5625                    && !external_tools.iter().any(|tool| tool.name == spec.name)
5626            })
5627            .collect::<Vec<_>>();
5628        specs.extend(external_tools.iter().cloned());
5629        specs
5630    }
5631
5632    fn task_ledger_tool_specs(
5633        &self,
5634        profile: Option<&ModelHarnessProfile>,
5635    ) -> Vec<roder_api::tools::ToolSpec> {
5636        self.tool_registry
5637            .get(TASK_LEDGER_TOOL_NAME)
5638            .map(|tool| {
5639                tool.spec()
5640                    .normalized_for_model_profile(schema_policy_for_model(profile))
5641            })
5642            .into_iter()
5643            .collect()
5644    }
5645
5646    pub(crate) fn engine_for(&self, provider: &str) -> anyhow::Result<Arc<dyn InferenceEngine>> {
5647        self.registry
5648            .inference_engine(provider)
5649            .or_else(|| {
5650                self.registry
5651                    .default_inference_engine()
5652                    .filter(|engine| provider.is_empty() || engine.id() == provider)
5653            })
5654            .ok_or_else(|| anyhow::anyhow!("inference provider {provider:?} is not registered"))
5655    }
5656
5657    pub async fn emit(&self, event: RoderEvent) -> EventEnvelope {
5658        self.dispatch_local_hook_lifecycle(&event).await;
5659        if let Some(record) = Self::lifecycle_record_for_event(&event) {
5660            let _ = self.persist_turn_lifecycle_record(&record).await;
5661        }
5662        let envelope = self.bus.emit(event);
5663        if let (Some(store), Some(thread_id)) = (&self.thread_store, envelope.thread_id.as_ref())
5664            && should_persist_thread_event(thread_id)
5665        {
5666            let _ = store.append_event(thread_id, &envelope).await;
5667        }
5668        // Registered event sinks (e.g. process extensions) receive the
5669        // persisted envelope through bounded per-sink queues; a slow sink
5670        // never blocks emit or turn progress.
5671        let dispatcher = self
5672            .event_sink_dispatcher
5673            .get_or_init(|| async {
5674                crate::event_sink_dispatch::EventSinkDispatcher::start(
5675                    &self.registry.event_sinks,
5676                    self.bus.clone(),
5677                )
5678            })
5679            .await;
5680        if !dispatcher.is_empty() {
5681            dispatcher.dispatch(&envelope, &self.bus);
5682        }
5683        envelope
5684    }
5685
5686    async fn dispatch_local_hook_lifecycle(&self, event: &RoderEvent) {
5687        // SessionStart belongs to the session, not the turn: firing it from
5688        // TurnStarted re-ran setup hooks on every user message. ThreadCreated
5689        // and ThreadLoaded carry no turn id, so they borrow a synthetic one the
5690        // way the SessionEnd path does.
5691        let session_turn_id = SESSION_HOOK_TURN_ID.to_string();
5692        let (thread_id, turn_id, name, matcher, input) = match event {
5693            RoderEvent::ThreadCreated(event) => (
5694                &event.thread_id,
5695                &session_turn_id,
5696                "SessionStart",
5697                Some("startup"),
5698                serde_json::json!({"source":"startup"}),
5699            ),
5700            RoderEvent::ThreadLoaded(event) => (
5701                &event.thread_id,
5702                &session_turn_id,
5703                "SessionStart",
5704                Some("resume"),
5705                serde_json::json!({"source":"resume"}),
5706            ),
5707            RoderEvent::TurnCompleted(event) => (
5708                &event.thread_id,
5709                &event.turn_id,
5710                "Stop",
5711                None,
5712                serde_json::json!({"finishReason":event.finish_reason}),
5713            ),
5714            RoderEvent::TurnFailed(event) => (
5715                &event.thread_id,
5716                &event.turn_id,
5717                "Stop",
5718                None,
5719                serde_json::json!({"error":event.error}),
5720            ),
5721            RoderEvent::ContextCompactionStarted(event) => (
5722                &event.thread_id,
5723                &event.turn_id,
5724                "PreCompact",
5725                None,
5726                serde_json::json!({"originalItemCount":event.original_item_count,"originalEstimatedTokens":event.original_estimated_tokens}),
5727            ),
5728            RoderEvent::ContextCompactionRecorded(event) => (
5729                &event.thread_id,
5730                &event.turn_id,
5731                "PostCompact",
5732                None,
5733                serde_json::json!({"compactedItemCount":event.compacted_item_count,"compactedEstimatedTokens":event.compacted_estimated_tokens}),
5734            ),
5735            RoderEvent::SubagentStarted(event) => (
5736                &event.parent_thread_id,
5737                &event.parent_turn_id,
5738                "SubagentStart",
5739                Some(event.agent_type.as_str()),
5740                serde_json::json!({"subagentThreadId":event.thread_id,"agentType":event.agent_type,"description":event.description}),
5741            ),
5742            RoderEvent::SubagentCompleted(event) => (
5743                &event.parent_thread_id,
5744                &event.parent_turn_id,
5745                "SubagentStop",
5746                Some(event.agent_type.as_str()),
5747                serde_json::json!({"subagentThreadId":event.thread_id,"agentType":event.agent_type,"exitReason":event.exit_reason}),
5748            ),
5749            RoderEvent::SubagentFailed(event) => (
5750                &event.parent_thread_id,
5751                &event.parent_turn_id,
5752                "SubagentStop",
5753                Some(event.agent_type.as_str()),
5754                serde_json::json!({"subagentThreadId":event.thread_id,"agentType":event.agent_type,"error":event.error}),
5755            ),
5756            _ => return,
5757        };
5758        // One SessionStart per session, whichever event opened it.
5759        if name == "SessionStart" && !self.claim_session_hook_start(thread_id).await {
5760            return;
5761        }
5762        let workspace = self.config.read().await.workspace.clone();
5763        crate::hooks::run_lifecycle(
5764            self,
5765            thread_id,
5766            turn_id,
5767            workspace.as_deref(),
5768            name,
5769            matcher,
5770            input,
5771        )
5772        .await;
5773    }
5774
5775    /// Claim the single `SessionStart` dispatch for a thread, returning whether
5776    /// this caller won it.
5777    pub(crate) async fn claim_session_hook_start(&self, thread_id: &ThreadId) -> bool {
5778        self.session_hook_started
5779            .write()
5780            .await
5781            .insert(thread_id.clone())
5782    }
5783
5784    /// Records a projected thread item event and persists it through the configured thread store.
5785    ///
5786    /// App-server protocol notification bridges currently call this after translating runtime
5787    /// events into item events. Headless runtime consumers that subscribe to `RoderEvent` directly
5788    /// must make the same call if they need the derived item event stream persisted.
5789    pub async fn record_thread_item_event_kind(
5790        &self,
5791        thread_id: &ThreadId,
5792        turn_id: &TurnId,
5793        timestamp: OffsetDateTime,
5794        kind: ThreadItemEventKind,
5795    ) -> anyhow::Result<ThreadItemEvent> {
5796        let seq = self.next_thread_item_event_seq(thread_id).await?;
5797        let item_event = ThreadItemEvent {
5798            seq,
5799            event_id: format!("{turn_id}-item-event-{seq}"),
5800            thread_id: thread_id.clone(),
5801            turn_id: turn_id.clone(),
5802            timestamp,
5803            event: kind,
5804        };
5805        if let Some(store) = &self.thread_store {
5806            store.append_item_event(thread_id, &item_event).await?;
5807        }
5808        self.remember_thread_item_event(&item_event).await?;
5809        Ok(item_event)
5810    }
5811
5812    async fn next_thread_item_event_seq(&self, thread_id: &ThreadId) -> anyhow::Result<u64> {
5813        self.ensure_thread_item_cache(thread_id).await?;
5814        Ok(self
5815            .thread_item_cache
5816            .lock()
5817            .await
5818            .next_item_event_seq(thread_id))
5819    }
5820
5821    pub async fn thread_item_exists(
5822        &self,
5823        thread_id: &ThreadId,
5824        turn_id: &TurnId,
5825        item_id: &str,
5826    ) -> anyhow::Result<bool> {
5827        self.ensure_thread_item_cache(thread_id).await?;
5828        Ok(self
5829            .thread_item_cache
5830            .lock()
5831            .await
5832            .thread_item_exists(thread_id, turn_id, item_id))
5833    }
5834
5835    pub async fn current_reasoning_item_id(
5836        &self,
5837        thread_id: &ThreadId,
5838        turn_id: &TurnId,
5839    ) -> anyhow::Result<Option<String>> {
5840        self.ensure_thread_item_cache(thread_id).await?;
5841        Ok(self
5842            .thread_item_cache
5843            .lock()
5844            .await
5845            .current_reasoning_item_id(thread_id, turn_id))
5846    }
5847
5848    async fn remember_thread_item_event(&self, item_event: &ThreadItemEvent) -> anyhow::Result<()> {
5849        self.ensure_thread_item_cache(&item_event.thread_id).await?;
5850        self.thread_item_cache
5851            .lock()
5852            .await
5853            .remember_item_event(item_event);
5854        Ok(())
5855    }
5856
5857    pub async fn latest_transcript_item_index(
5858        &self,
5859        thread_id: &ThreadId,
5860        turn_id: &TurnId,
5861    ) -> anyhow::Result<Option<usize>> {
5862        self.ensure_thread_item_cache(thread_id).await?;
5863        Ok(self
5864            .thread_item_cache
5865            .lock()
5866            .await
5867            .latest_transcript_item_index(thread_id, turn_id))
5868    }
5869
5870    async fn next_transcript_item_index(
5871        &self,
5872        thread_id: &ThreadId,
5873        turn_id: &TurnId,
5874    ) -> anyhow::Result<usize> {
5875        self.ensure_thread_item_cache(thread_id).await?;
5876        Ok(self
5877            .thread_item_cache
5878            .lock()
5879            .await
5880            .next_transcript_item_index(thread_id, turn_id))
5881    }
5882
5883    async fn remember_transcript_item_index(
5884        &self,
5885        thread_id: &ThreadId,
5886        turn_id: &TurnId,
5887        item_index: usize,
5888    ) -> anyhow::Result<()> {
5889        self.ensure_thread_item_cache(thread_id).await?;
5890        self.thread_item_cache
5891            .lock()
5892            .await
5893            .remember_transcript_item_index(thread_id, turn_id, item_index);
5894        Ok(())
5895    }
5896
5897    async fn ensure_thread_item_cache(&self, thread_id: &ThreadId) -> anyhow::Result<()> {
5898        if self
5899            .thread_item_cache
5900            .lock()
5901            .await
5902            .contains_thread(thread_id)
5903        {
5904            return Ok(());
5905        }
5906
5907        let snapshot = if let Some(store) = &self.thread_store {
5908            store.load_thread(thread_id).await?
5909        } else {
5910            None
5911        };
5912        self.thread_item_cache.lock().await.ensure_thread(
5913            thread_id,
5914            ThreadItemCacheEntry::from_snapshot(snapshot.as_ref()),
5915        );
5916        Ok(())
5917    }
5918
5919    pub(crate) async fn persist_turn_item(
5920        &self,
5921        thread_id: &ThreadId,
5922        turn_id: &TurnId,
5923        item: &TranscriptItem,
5924    ) -> anyhow::Result<()> {
5925        let item_index = self.next_transcript_item_index(thread_id, turn_id).await?;
5926        let timestamp = OffsetDateTime::now_utc();
5927        self.emit(RoderEvent::TranscriptItemAppended(TranscriptItemAppended {
5928            thread_id: thread_id.clone(),
5929            turn_id: turn_id.clone(),
5930            item_type: match item {
5931                TranscriptItem::UserMessage(_) => "user_message",
5932                TranscriptItem::AssistantMessage(_) => "assistant_message",
5933                TranscriptItem::ReasoningSummary(_) => "reasoning_summary",
5934                TranscriptItem::ToolCall(_) => "tool_call",
5935                TranscriptItem::ToolResult(_) => "tool_result",
5936                TranscriptItem::FileChange(_) => "file_change",
5937                TranscriptItem::ContextCompaction(_) => "context_compaction",
5938                TranscriptItem::Error(_) => "error",
5939                TranscriptItem::ProviderMetadata(_) => "provider_metadata",
5940            }
5941            .to_string(),
5942            item_index: Some(item_index),
5943            item: Some(item.clone()),
5944            timestamp,
5945        }))
5946        .await;
5947        self.remember_transcript_item_index(thread_id, turn_id, item_index)
5948            .await?;
5949        Ok(())
5950    }
5951}
5952
5953fn transcript_has_images(transcript: &[TranscriptItem]) -> bool {
5954    transcript.iter().any(|item| {
5955        matches!(
5956            item,
5957            TranscriptItem::UserMessage(message) if !message.images.is_empty()
5958        )
5959    })
5960}
5961
5962fn transcript_has_task_ledger(transcript: &[TranscriptItem]) -> bool {
5963    transcript.iter().any(|item| {
5964        matches!(
5965            item,
5966            TranscriptItem::ToolResult(result)
5967                if result.name.as_deref() == Some(TASK_LEDGER_TOOL_NAME) && !result.is_error
5968        )
5969    })
5970}
5971
5972fn task_ledger_completion_prompt(transcript: &[TranscriptItem]) -> Option<String> {
5973    let latest = transcript.iter().rev().find_map(|item| match item {
5974        TranscriptItem::ToolResult(result)
5975            if result.name.as_deref() == Some(TASK_LEDGER_TOOL_NAME) && !result.is_error =>
5976        {
5977            Some(result.result.as_str())
5978        }
5979        _ => None,
5980    })?;
5981    if !task_ledger_has_open_items(latest) {
5982        return None;
5983    }
5984
5985    let mut ledger = latest.chars().take(1500).collect::<String>();
5986    if latest.chars().nth(1500).is_some() {
5987        ledger.push_str("...");
5988    }
5989    Some(format!(
5990        "Task Ledger Completion Required: the latest task ledger still has pending or in-progress items. Do not provide a final answer yet. Use tools to complete the remaining scoreable work, create or update any required output files, then call `{TASK_LEDGER_TOOL_NAME}` with every task completed and evidence before finalizing.\n\nLatest ledger:\n{ledger}"
5991    ))
5992}
5993
5994fn task_ledger_deadline_completion_prompt(
5995    remaining_seconds: u64,
5996    reserve_seconds: u64,
5997    completion_prompt: &str,
5998) -> String {
5999    format!(
6000        "Eval deadline scoreable completion: {remaining_seconds} seconds remain in the {reserve_seconds}-second finalization reserve. Do not browse, search, or start slow work. Use the available tools now to create or update the required scoreable output files, run only a quick local check if needed, then update the task ledger to completed before finalizing.\n\n{completion_prompt}"
6001    )
6002}
6003
6004fn task_ledger_scoreable_checkpoint_prompt(
6005    remaining_seconds: u64,
6006    completion_prompt: &str,
6007) -> String {
6008    format!(
6009        "Scoreable Output Checkpoint: {remaining_seconds} seconds remain before the eval deadline. Before any further research, browsing, or long commands, use tools now to ensure the required output file(s) exist with the best evidence-backed answer, even if provisional. If a scoreable file already exists, read it and preserve that candidate unless you have stronger task-specific evidence for a replacement. Do not overwrite a plausible dated, historical, or local-evidence candidate with a current live-page, partial-coverage, or weaker guess merely to refresh the checkpoint. You may continue refining afterward, but do not apologize or finalize until the scoreable file exists and the task ledger is updated.\n\n{completion_prompt}"
6010    )
6011}
6012
6013fn task_ledger_has_open_items(ledger: &str) -> bool {
6014    ledger.lines().any(|line| {
6015        let line = line.trim_start();
6016        line.starts_with("- pending:") || line.starts_with("- in_progress:")
6017    })
6018}
6019
6020fn turn_deadline_for_config(cfg: &RuntimeConfig) -> Option<OffsetDateTime> {
6021    cfg.turn_deadline_seconds
6022        .filter(|seconds| *seconds > 0)
6023        .map(|seconds| OffsetDateTime::now_utc() + Duration::seconds(seconds as i64))
6024}
6025
6026fn deadline_expired(deadline: OffsetDateTime) -> bool {
6027    OffsetDateTime::now_utc() >= deadline
6028}
6029
6030pub(crate) fn deadline_remaining_seconds(deadline: Option<OffsetDateTime>) -> Option<u64> {
6031    let deadline = deadline?;
6032    if deadline <= OffsetDateTime::now_utc() {
6033        return Some(0);
6034    }
6035    Some(
6036        (deadline - OffsetDateTime::now_utc())
6037            .unsigned_abs()
6038            .as_secs()
6039            .max(1),
6040    )
6041}
6042
6043fn deadline_instant(deadline: OffsetDateTime) -> tokio::time::Instant {
6044    let now = OffsetDateTime::now_utc();
6045    if deadline <= now {
6046        return tokio::time::Instant::now();
6047    }
6048    tokio::time::Instant::now() + (deadline - now).unsigned_abs()
6049}
6050
6051fn inference_timeout_deadline(
6052    deadline: Option<OffsetDateTime>,
6053    runtime_profile: RuntimeProfile,
6054    task_ledger_required: bool,
6055    reserve_seconds: u64,
6056    finalization_requested: bool,
6057    task_ledger_scoreable_checkpoints: u8,
6058    transcript: &[TranscriptItem],
6059) -> Option<(OffsetDateTime, InferenceTimeoutAction)> {
6060    let deadline = deadline?;
6061    if runtime_profile == RuntimeProfile::Eval
6062        && task_ledger_required
6063        && !finalization_requested
6064        && task_ledger_scoreable_checkpoints < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
6065        && TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS > reserve_seconds
6066        && task_ledger_completion_prompt(transcript).is_some()
6067    {
6068        let checkpoint_deadline =
6069            deadline - Duration::seconds(TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS as i64);
6070        if checkpoint_deadline > OffsetDateTime::now_utc() {
6071            return Some((
6072                checkpoint_deadline,
6073                InferenceTimeoutAction::ScoreableCheckpoint,
6074            ));
6075        }
6076    }
6077    if runtime_profile == RuntimeProfile::Eval && !finalization_requested {
6078        return Some((
6079            deadline - Duration::seconds(reserve_seconds as i64),
6080            InferenceTimeoutAction::Finalization,
6081        ));
6082    }
6083    Some((deadline, InferenceTimeoutAction::Finalization))
6084}
6085
6086fn turn_partial_result(transcript: &[TranscriptItem]) -> String {
6087    let tool_results = transcript
6088        .iter()
6089        .filter(|item| matches!(item, TranscriptItem::ToolResult(_)))
6090        .count();
6091    let assistant_messages = transcript
6092        .iter()
6093        .filter(|item| matches!(item, TranscriptItem::AssistantMessage(_)))
6094        .count();
6095    format!(
6096        "partial turn state: {} transcript items, {assistant_messages} assistant messages, {tool_results} tool results",
6097        transcript.len()
6098    )
6099}
6100
6101fn reasoning_for_model(cfg: &RuntimeConfig, model: &str) -> ReasoningConfig {
6102    let level = effective_reasoning_for_model(cfg, model);
6103    match level.as_str() {
6104        "" | REASONING_NONE => ReasoningConfig::default(),
6105        level => ReasoningConfig {
6106            enabled: true,
6107            level: Some(level.to_string()),
6108        },
6109    }
6110}
6111
6112fn server_side_compaction_threshold(cfg: &RuntimeConfig, model: &str) -> Option<u32> {
6113    let entry = lookup_model(model)?;
6114    if !entry.supports_compaction {
6115        return None;
6116    }
6117    cfg.auto_compact_token_limit
6118        .or_else(|| {
6119            model_profile_for_model(cfg, model).and_then(|profile| profile.auto_compact_token_limit)
6120        })
6121        .or(Some(entry.auto_compact_token_limit))
6122        .filter(|threshold| *threshold > 0)
6123}
6124
6125pub(crate) fn tool_search_for_provider_model(
6126    cfg: &RuntimeConfig,
6127    provider: &str,
6128    model: &str,
6129) -> ToolSearchConfig {
6130    let mut resolved = cfg.tool_search.clone();
6131    if let Some(provider_config) = cfg.provider_tool_search.get(provider) {
6132        provider_config.apply_to(&mut resolved);
6133    }
6134    if let Some(model_config) = cfg.model_tool_search.get(model) {
6135        model_config.apply_to(&mut resolved);
6136    }
6137    resolved
6138}
6139
6140fn parallel_tool_calls_for_model(cfg: &RuntimeConfig, model: &str) -> bool {
6141    cfg.model_parallel_tool_calls
6142        .get(model)
6143        .copied()
6144        .or_else(|| {
6145            model_profile_for_model(cfg, model).and_then(|profile| profile.parallel_tool_calls)
6146        })
6147        .unwrap_or(true)
6148}
6149
6150/// Count the children an `agent_swarm` call will launch from its arguments
6151/// (item-based spawns plus resumes). Lenient: returns 0 on malformed input.
6152fn agent_swarm_child_count(arguments: &str) -> usize {
6153    serde_json::from_str::<roder_api::subagents::AgentSwarmRequest>(arguments)
6154        .map(|request| request.items.len() + request.resume_agent_ids.len())
6155        .unwrap_or(0)
6156}
6157
6158/// Parse `completed`/`failed`/`aborted` counts from an `<agent_swarm_result>`
6159/// summary line, e.g. `<summary>completed: 2, failed: 1</summary>`. Omitted
6160/// buckets default to 0. Returns `None` when the text is not a swarm result.
6161fn parse_swarm_counts(text: &str) -> Option<(usize, usize, usize)> {
6162    if !text.contains("<agent_swarm_result>") {
6163        return None;
6164    }
6165    let bucket = |label: &str| -> usize {
6166        let needle = format!("{label}: ");
6167        text.find(&needle)
6168            .map(|start| start + needle.len())
6169            .map(|start| {
6170                text[start..]
6171                    .chars()
6172                    .take_while(|c| c.is_ascii_digit())
6173                    .collect::<String>()
6174            })
6175            .and_then(|digits| digits.parse().ok())
6176            .unwrap_or(0)
6177    };
6178    Some((bucket("completed"), bucket("failed"), bucket("aborted")))
6179}
6180
6181fn effective_reasoning_for_model(cfg: &RuntimeConfig, model: &str) -> String {
6182    let base_reasoning = default_effective_reasoning_for_model(cfg, model);
6183    if cfg.dynamic_workflows.effort_profile == DynamicWorkflowEffortProfile::Ultracode {
6184        return ultracode_reasoning_level_for_model(
6185            model,
6186            &cfg.speed_policy.ultracode_reasoning,
6187            &base_reasoning,
6188        );
6189    }
6190    base_reasoning
6191}
6192
6193fn default_effective_reasoning_for_model(cfg: &RuntimeConfig, model: &str) -> String {
6194    let Some(entry) = lookup_model(model) else {
6195        return cfg
6196            .reasoning
6197            .clone()
6198            .unwrap_or_else(|| REASONING_NONE.to_string());
6199    };
6200    if entry.supported_reasoning.is_empty() {
6201        return REASONING_NONE.to_string();
6202    }
6203    cfg.reasoning
6204        .as_deref()
6205        .filter(|reasoning| {
6206            entry
6207                .supported_reasoning
6208                .iter()
6209                .any(|option| option.effort == *reasoning)
6210        })
6211        .map(str::to_string)
6212        .or_else(|| {
6213            model_profile_for_model(cfg, model)
6214                .and_then(|profile| profile.reasoning.orientation)
6215                .filter(|reasoning| {
6216                    entry
6217                        .supported_reasoning
6218                        .iter()
6219                        .any(|option| option.effort == reasoning)
6220                })
6221        })
6222        .unwrap_or_else(|| entry.default_reasoning.to_string())
6223}
6224
6225fn validate_reasoning_effort(model: &str, effort: &str) -> anyhow::Result<()> {
6226    if effort == REASONING_NONE && !model_supports_reasoning(model, effort) {
6227        return Ok(());
6228    }
6229    let Some(entry) = lookup_model(model) else {
6230        return Ok(());
6231    };
6232    if entry
6233        .supported_reasoning
6234        .iter()
6235        .any(|option| option.effort == effort)
6236    {
6237        Ok(())
6238    } else {
6239        anyhow::bail!("model {model} does not support reasoning effort {effort}")
6240    }
6241}
6242
6243fn validate_runtime_config_reasoning(cfg: &RuntimeConfig) -> anyhow::Result<()> {
6244    let Some(reasoning) = cfg.reasoning.as_deref() else {
6245        return Ok(());
6246    };
6247    let Some(entry) = lookup_model(&cfg.default_model) else {
6248        return Ok(());
6249    };
6250    if entry.provider != PROVIDER_GEMINI {
6251        return Ok(());
6252    }
6253    validate_reasoning_effort(&cfg.default_model, reasoning)
6254}
6255
6256fn validate_runtime_inference_router_config(
6257    registry: &ExtensionRegistry,
6258    cfg: &RuntimeConfig,
6259) -> anyhow::Result<()> {
6260    if !cfg.inference_router.enabled {
6261        return Ok(());
6262    }
6263    let Some(router_id) = cfg.inference_router.router_id.as_deref() else {
6264        anyhow::bail!("inference_router.enabled requires inference_router.router");
6265    };
6266    if registry.inference_router(router_id).is_some() {
6267        return Ok(());
6268    }
6269    let available = registry
6270        .inference_routers
6271        .iter()
6272        .map(|router| router.id())
6273        .collect::<Vec<_>>()
6274        .join(", ");
6275    if available.is_empty() {
6276        anyhow::bail!("inference router {router_id:?} is not registered");
6277    }
6278    anyhow::bail!(
6279        "inference router {router_id:?} is not registered; available routers: {available}"
6280    );
6281}
6282
6283fn model_supports_reasoning(model: &str, effort: &str) -> bool {
6284    lookup_model(model)
6285        .map(|entry| {
6286            entry
6287                .supported_reasoning
6288                .iter()
6289                .any(|option| option.effort == effort)
6290        })
6291        .unwrap_or(false)
6292}
6293
6294fn is_final_answer_phase(phase: Option<&str>) -> bool {
6295    phase.is_none_or(|phase| phase.is_empty() || phase == FINAL_ANSWER_PHASE)
6296}
6297
6298fn edit_tool_for_model<'a>(cfg: &'a RuntimeConfig, model: &'a str) -> Option<&'a str> {
6299    cfg.model_edit_tools
6300        .get(model)
6301        .map(String::as_str)
6302        .or_else(|| {
6303            cfg.model_profiles
6304                .get(model)
6305                .and_then(|profile| profile.edit_tool.as_deref())
6306        })
6307        .or_else(|| lookup_model(model).and_then(|entry| entry.edit_tool))
6308        .or(Some(EDIT_TOOL_EDIT))
6309}
6310
6311fn model_profile_for_model(cfg: &RuntimeConfig, model: &str) -> Option<ModelHarnessProfile> {
6312    cfg.model_profiles
6313        .get(model)
6314        .cloned()
6315        .or_else(|| built_in_model_profile(model))
6316}
6317
6318pub(crate) fn allowlist_permits(allowlist: &[String], tool_name: &str) -> bool {
6319    allowlist.is_empty() || allowlist.iter().any(|allowed| allowed == tool_name)
6320}
6321
6322/// Provider-aware profile resolution for the active turn.
6323///
6324/// Many model ids are shared across providers (for example Cursor proxies
6325/// `claude-opus-4-8`). Resolving the harness profile by id alone picks the
6326/// first catalog entry, which assigns cross-provider ids the wrong family and
6327/// instruction overlay (e.g. a `cursor/claude-opus-4-8` turn would otherwise
6328/// inherit the Anthropic overlay). Prefer the explicit `(provider, model)`
6329/// catalog entry, keeping user-configured profile overrides as the top
6330/// precedence.
6331fn model_profile_for_provider_model(
6332    cfg: &RuntimeConfig,
6333    provider: &str,
6334    model: &str,
6335) -> Option<ModelHarnessProfile> {
6336    cfg.model_profiles
6337        .get(model)
6338        .cloned()
6339        .or_else(|| built_in_model_profile_for_provider(provider, model))
6340}
6341
6342fn schema_policy_for_model(profile: Option<&ModelHarnessProfile>) -> ModelSchemaPolicy {
6343    profile
6344        .map(|profile| profile.schema_policy)
6345        .unwrap_or_default()
6346}
6347
6348fn model_profile_segment_metadata(
6349    profile: Option<&ModelHarnessProfile>,
6350    provider: &str,
6351    model: &str,
6352    segment: &str,
6353) -> serde_json::Value {
6354    serde_json::json!({
6355        "kind": MODEL_PROFILE_TRACE_KIND,
6356        "segment": segment,
6357        "provider": provider,
6358        "model": model,
6359        "profileModel": profile.map(|profile| profile.model.as_str()).unwrap_or(model),
6360        "providerFamily": profile.map(|profile| profile.provider_family),
6361        "editTool": profile.and_then(|profile| profile.edit_tool.as_deref()),
6362        "schemaPolicy": profile.map(|profile| profile.schema_policy),
6363        "instructionOverlay": profile.map(|profile| profile.instruction_overlay),
6364        "parallelToolCalls": profile.and_then(|profile| profile.parallel_tool_calls),
6365        "autoCompactTokenLimit": profile.and_then(|profile| profile.auto_compact_token_limit),
6366    })
6367}
6368
6369fn model_switch_summary(
6370    transcript: &[TranscriptItem],
6371    profile: Option<&ModelHarnessProfile>,
6372    provider: &str,
6373    model: &str,
6374    tools: &[roder_api::tools::ToolSpec],
6375) -> Option<String> {
6376    let previous = latest_model_profile_segment(transcript)?;
6377    let previous_model = previous
6378        .get("model")
6379        .and_then(serde_json::Value::as_str)
6380        .unwrap_or_default();
6381    let previous_provider = previous
6382        .get("provider")
6383        .and_then(serde_json::Value::as_str)
6384        .unwrap_or_default();
6385    if previous_model == model && previous_provider == provider {
6386        return None;
6387    }
6388
6389    let previous_profile = previous
6390        .get("profileModel")
6391        .and_then(serde_json::Value::as_str)
6392        .unwrap_or(previous_model);
6393    let current_profile = profile
6394        .map(|profile| profile.model.as_str())
6395        .unwrap_or(model);
6396    let previous_edit_tool = previous
6397        .get("editTool")
6398        .and_then(serde_json::Value::as_str)
6399        .unwrap_or("none");
6400    let current_edit_tool = profile
6401        .and_then(|profile| profile.edit_tool.as_deref())
6402        .unwrap_or("none");
6403    let tool_names = tools
6404        .iter()
6405        .map(|tool| tool.name.as_str())
6406        .take(12)
6407        .collect::<Vec<_>>()
6408        .join(", ");
6409    Some(format!(
6410        "{MODEL_SWITCH_SUMMARY_PREFIX} previous profile {previous_provider}/{previous_profile} used edit tool {previous_edit_tool}. Current profile {provider}/{current_profile} uses edit tool {current_edit_tool}. Available tools now: {}.",
6411        if tool_names.is_empty() {
6412            "none"
6413        } else {
6414            &tool_names
6415        }
6416    ))
6417}
6418
6419fn latest_model_profile_segment(transcript: &[TranscriptItem]) -> Option<&serde_json::Value> {
6420    transcript.iter().rev().find_map(|item| {
6421        let TranscriptItem::ProviderMetadata(value) = item else {
6422            return None;
6423        };
6424        (value.get("kind").and_then(serde_json::Value::as_str) == Some(MODEL_PROFILE_TRACE_KIND))
6425            .then_some(value)
6426    })
6427}
6428
6429pub fn validate_edit_tool(value: &str) -> anyhow::Result<()> {
6430    match value.trim() {
6431        EDIT_TOOL_PATCH | EDIT_TOOL_EDIT => Ok(()),
6432        _ => anyhow::bail!(
6433            "unsupported edit_tool {value:?}; allowed values: {EDIT_TOOL_PATCH}, {EDIT_TOOL_EDIT}"
6434        ),
6435    }
6436}
6437
6438fn should_persist_thread_event(thread_id: &str) -> bool {
6439    !is_synthetic_event_thread_id(thread_id)
6440}
6441
6442#[cfg(test)]
6443#[path = "runtime/codex_v2_tests.rs"]
6444mod codex_v2_tests;
6445
6446#[cfg(test)]
6447#[path = "runtime/codex_v2_lifecycle_tests.rs"]
6448mod codex_v2_lifecycle_tests;
6449
6450#[cfg(test)]
6451mod tests {
6452    use super::*;
6453    use futures::stream;
6454    use roder_api::catalog::{
6455        PROVIDER_MOCK, REASONING_HIGH, REASONING_LOW, REASONING_MEDIUM, REASONING_MINIMAL,
6456        REASONING_NONE, REASONING_XHIGH,
6457    };
6458    use roder_api::extension::ExtensionRegistryBuilder;
6459    use roder_api::inference::{
6460        CompletionMetadata, InferenceCapabilities, InferenceEngine, InferenceEventStream,
6461        InferenceProviderContext, InferenceTurnContext, MessageDelta, ModelDescriptor,
6462        ModelInstructionOverlay, ModelProfileReasoning, ModelSchemaPolicy, ProviderFamily,
6463        ReasoningEffortDescriptor,
6464    };
6465    use roder_api::inference_routing::{
6466        InferenceRouter, InferenceRoutingContext, InferenceRoutingDecision, InferenceRoutingOutcome,
6467    };
6468    use roder_api::thread::ThreadStoreFactory;
6469    use roder_api::tools::{ToolContributor, ToolExecutor, ToolSpec};
6470    use roder_ext_jsonl_thread_store::store::JsonlThreadStoreFactory;
6471    use std::sync::Mutex as StdMutex;
6472
6473    fn test_workspace() -> String {
6474        std::env::current_dir().unwrap().display().to_string()
6475    }
6476
6477    #[test]
6478    fn interactive_explicit_turn_deadline_has_bounded_remaining_time() {
6479        let config = RuntimeConfig {
6480            runtime_profile: RuntimeProfile::Interactive,
6481            turn_deadline_seconds: Some(60),
6482            ..RuntimeConfig::default()
6483        };
6484
6485        let deadline = turn_deadline_for_config(&config).expect("configured turn deadline");
6486        let remaining = deadline_remaining_seconds(Some(deadline));
6487
6488        assert!(
6489            matches!(remaining, Some(1..=60)),
6490            "interactive deadline must be positive and no longer than configured: {remaining:?}"
6491        );
6492    }
6493
6494    #[test]
6495    fn interactive_without_turn_deadline_stays_unbounded() {
6496        let config = RuntimeConfig {
6497            runtime_profile: RuntimeProfile::Interactive,
6498            turn_deadline_seconds: None,
6499            ..RuntimeConfig::default()
6500        };
6501
6502        assert_eq!(turn_deadline_for_config(&config), None);
6503    }
6504
6505    struct MetadataMissingStore;
6506
6507    #[async_trait::async_trait]
6508    impl ThreadStore for MetadataMissingStore {
6509        fn id(&self) -> roder_api::thread::ThreadStoreId {
6510            "metadata-missing-store".to_string()
6511        }
6512
6513        async fn create_thread(&self, metadata: ThreadMetadata) -> anyhow::Result<ThreadMetadata> {
6514            Ok(metadata)
6515        }
6516
6517        async fn list_threads(&self) -> anyhow::Result<Vec<ThreadMetadata>> {
6518            Ok(Vec::new())
6519        }
6520
6521        async fn load_thread(
6522            &self,
6523            _thread_id: &ThreadId,
6524        ) -> anyhow::Result<Option<ThreadSnapshot>> {
6525            Ok(Some(ThreadSnapshot {
6526                metadata: None,
6527                ..ThreadSnapshot::default()
6528            }))
6529        }
6530
6531        async fn append_event(
6532            &self,
6533            _thread_id: &ThreadId,
6534            _envelope: &EventEnvelope,
6535        ) -> anyhow::Result<()> {
6536            Ok(())
6537        }
6538    }
6539
6540    struct MetadataMissingStoreFactory;
6541
6542    impl ThreadStoreFactory for MetadataMissingStoreFactory {
6543        fn id(&self) -> roder_api::thread::ThreadStoreId {
6544            "metadata-missing-store".to_string()
6545        }
6546
6547        fn create(&self) -> Arc<dyn ThreadStore> {
6548            Arc::new(MetadataMissingStore)
6549        }
6550    }
6551
6552    #[test]
6553    fn synthetic_app_server_events_are_not_thread_events() {
6554        for thread_id in ["app-server", "runtime", "thread-workflow"] {
6555            assert!(!should_persist_thread_event(thread_id));
6556        }
6557        assert!(should_persist_thread_event("thread-discovery"));
6558        assert!(should_persist_thread_event("thread-plan"));
6559        assert!(should_persist_thread_event("thread-process"));
6560        assert!(should_persist_thread_event("thread-1"));
6561    }
6562
6563    #[test]
6564    fn server_side_compaction_uses_catalog_ninety_percent_default() {
6565        assert_eq!(
6566            server_side_compaction_threshold(&RuntimeConfig::default(), "gpt-5.5"),
6567            Some(945_000)
6568        );
6569        assert_eq!(
6570            server_side_compaction_threshold(&RuntimeConfig::default(), "gpt-5.3-codex-spark"),
6571            Some(115_200)
6572        );
6573    }
6574
6575    #[test]
6576    fn server_side_compaction_respects_explicit_config_override() {
6577        let cfg = RuntimeConfig {
6578            auto_compact_token_limit: Some(123_456),
6579            ..RuntimeConfig::default()
6580        };
6581
6582        assert_eq!(
6583            server_side_compaction_threshold(&cfg, "gpt-5.5"),
6584            Some(123_456)
6585        );
6586    }
6587
6588    #[tokio::test]
6589    async fn pre_request_compaction_runs_when_server_side_model_is_at_context_window() {
6590        let captured = Arc::new(StdMutex::new(None));
6591        let mut builder = ExtensionRegistryBuilder::new();
6592        builder.inference_engine(Arc::new(CapturingEngine {
6593            request: captured.clone(),
6594        }));
6595        let thread_root = std::env::temp_dir().join(format!(
6596            "roder-pre-request-compaction-{}",
6597            uuid::Uuid::new_v4()
6598        ));
6599        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
6600            base_path: thread_root.clone(),
6601        }));
6602        let runtime = Arc::new(
6603            Runtime::new(
6604                builder.build().unwrap(),
6605                RuntimeConfig {
6606                    default_provider: PROVIDER_MOCK.to_string(),
6607                    default_model: "gpt-5.5".to_string(),
6608                    file_backed_dynamic_context: true,
6609                    ..RuntimeConfig::default()
6610                },
6611            )
6612            .unwrap(),
6613        );
6614        let thread_id = runtime
6615            .create_thread(Some("Pre-request compaction".to_string()))
6616            .await
6617            .unwrap()
6618            .thread_id;
6619        let old_turn = "old-turn".to_string();
6620        runtime
6621            .persist_turn_item(
6622                &thread_id,
6623                &old_turn,
6624                &TranscriptItem::UserMessage(UserMessage::text("old context ".repeat(4_300_000))),
6625            )
6626            .await
6627            .unwrap();
6628
6629        let mut events = runtime.subscribe_events();
6630        runtime
6631            .start_turn(StartTurnRequest {
6632                thread_id: thread_id.clone(),
6633                message: "continue".to_string(),
6634                images: Vec::new(),
6635                provider_override: None,
6636                model_override: None,
6637                reasoning_override: None,
6638                workspace: test_workspace(),
6639                instructions: InstructionBundle::default(),
6640                developer_context: None,
6641                task_ledger_required: false,
6642                service_tier_override: None,
6643            })
6644            .await
6645            .unwrap();
6646        loop {
6647            let envelope = tokio::time::timeout(std::time::Duration::from_secs(5), events.recv())
6648                .await
6649                .unwrap()
6650                .unwrap();
6651            if envelope.thread_id.as_deref() == Some(&thread_id)
6652                && matches!(envelope.event, RoderEvent::TurnCompleted(_))
6653            {
6654                break;
6655            }
6656        }
6657
6658        let request = captured.lock().unwrap().clone().unwrap();
6659        assert!(
6660            matches!(
6661                request.transcript.first(),
6662                Some(TranscriptItem::ContextCompaction(_))
6663            ),
6664            "provider request should start with a local emergency compaction item"
6665        );
6666        assert!(
6667            request.transcript.len() < 4,
6668            "provider request should not replay the full oversized prior transcript: {:?}",
6669            request.transcript
6670        );
6671
6672        let _ = std::fs::remove_dir_all(thread_root);
6673    }
6674
6675    #[tokio::test]
6676    async fn continue_after_context_window_failure_compacts_before_provider_request() {
6677        let captured = Arc::new(StdMutex::new(None));
6678        let mut builder = ExtensionRegistryBuilder::new();
6679        builder.inference_engine(Arc::new(CapturingEngine {
6680            request: captured.clone(),
6681        }));
6682        let thread_root = std::env::temp_dir().join(format!(
6683            "roder-context-failure-continue-{}",
6684            uuid::Uuid::new_v4()
6685        ));
6686        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
6687            base_path: thread_root.clone(),
6688        }));
6689        let runtime = Arc::new(
6690            Runtime::new(
6691                builder.build().unwrap(),
6692                RuntimeConfig {
6693                    default_provider: PROVIDER_MOCK.to_string(),
6694                    default_model: "gpt-5.5".to_string(),
6695                    file_backed_dynamic_context: true,
6696                    ..RuntimeConfig::default()
6697                },
6698            )
6699            .unwrap(),
6700        );
6701        let thread_id = runtime
6702            .create_thread(Some("Context failure continue".to_string()))
6703            .await
6704            .unwrap()
6705            .thread_id;
6706        let failed_turn = "failed-turn".to_string();
6707        runtime
6708            .persist_turn_item(
6709                &thread_id,
6710                &failed_turn,
6711                &TranscriptItem::UserMessage(UserMessage::text("old work ".repeat(10_000))),
6712            )
6713            .await
6714            .unwrap();
6715        runtime
6716            .persist_turn_item(
6717                &thread_id,
6718                &failed_turn,
6719                &TranscriptItem::Error(ErrorRecord {
6720                    message: "Your input exceeds the context window of this model. Please adjust your input and try again."
6721                        .to_string(),
6722                }),
6723            )
6724            .await
6725            .unwrap();
6726
6727        let mut events = runtime.subscribe_events();
6728        runtime
6729            .start_turn(StartTurnRequest {
6730                thread_id: thread_id.clone(),
6731                message: "continue".to_string(),
6732                images: Vec::new(),
6733                provider_override: None,
6734                model_override: None,
6735                reasoning_override: None,
6736                workspace: test_workspace(),
6737                instructions: InstructionBundle::default(),
6738                developer_context: None,
6739                task_ledger_required: false,
6740                service_tier_override: None,
6741            })
6742            .await
6743            .unwrap();
6744        loop {
6745            let envelope = tokio::time::timeout(std::time::Duration::from_secs(5), events.recv())
6746                .await
6747                .unwrap()
6748                .unwrap();
6749            if envelope.thread_id.as_deref() == Some(&thread_id)
6750                && matches!(envelope.event, RoderEvent::TurnCompleted(_))
6751            {
6752                break;
6753            }
6754        }
6755
6756        let request = captured.lock().unwrap().clone().unwrap();
6757        assert!(
6758            matches!(
6759                request.transcript.first(),
6760                Some(TranscriptItem::ContextCompaction(_))
6761            ),
6762            "provider request after context-window failure should start with local compaction"
6763        );
6764        assert!(
6765            request
6766                .transcript
6767                .iter()
6768                .any(|item| matches!(item, TranscriptItem::UserMessage(message) if message.text == "continue")),
6769            "current continue prompt must be preserved: {:?}",
6770            request.transcript
6771        );
6772        assert!(
6773            !request.transcript.iter().any(
6774                |item| matches!(item, TranscriptItem::Error(error) if error.message.contains("context window"))
6775            ),
6776            "raw prior context-window error should be summarized, not replayed: {:?}",
6777            request.transcript
6778        );
6779
6780        let _ = std::fs::remove_dir_all(thread_root);
6781    }
6782
6783    #[tokio::test]
6784    async fn workspace_for_thread_falls_back_when_metadata_is_missing() {
6785        let workspace = test_workspace();
6786        let mut builder = ExtensionRegistryBuilder::new();
6787        builder.inference_engine(Arc::new(FakeInferenceEngine));
6788        builder.thread_store_factory(Arc::new(MetadataMissingStoreFactory));
6789        let runtime = Runtime::new(
6790            builder.build().unwrap(),
6791            RuntimeConfig {
6792                workspace: Some(workspace.clone()),
6793                ..RuntimeConfig::default()
6794            },
6795        )
6796        .unwrap();
6797
6798        let resolved = runtime
6799            .workspace_for_thread(&ThreadId::from("thread-workflow"))
6800            .await
6801            .unwrap();
6802
6803        assert_eq!(resolved, workspace);
6804    }
6805
6806    #[tokio::test]
6807    async fn automations_can_create_project_thread_with_model_overrides() {
6808        let runtime = Runtime::fake().unwrap();
6809        let workspace = std::env::temp_dir().join("project");
6810        let metadata = runtime
6811            .create_thread_with(CreateThreadRequest {
6812                title: Some("Automation: nightly status".to_string()),
6813                workspace: workspace.display().to_string(),
6814                workspace_id: None,
6815                root_id: None,
6816                provider: Some("mock".to_string()),
6817                model: Some("mock".to_string()),
6818                selection_mode: None,
6819                tool_allowlist: Vec::new(),
6820                developer_instructions: None,
6821                external_tools: Vec::new(),
6822                runner: None,
6823            })
6824            .await
6825            .unwrap();
6826
6827        assert_eq!(
6828            metadata.title.as_deref(),
6829            Some("Automation: nightly status")
6830        );
6831        assert_eq!(metadata.workspace, workspace.display().to_string());
6832        assert_eq!(metadata.provider.as_deref(), Some("mock"));
6833        assert_eq!(metadata.model.as_deref(), Some("mock"));
6834    }
6835
6836    #[tokio::test]
6837    async fn prior_provider_compaction_boundary_is_used_on_next_turn() {
6838        let captured = Arc::new(StdMutex::new(None));
6839        let mut builder = ExtensionRegistryBuilder::new();
6840        builder.inference_engine(Arc::new(CapturingEngine {
6841            request: captured.clone(),
6842        }));
6843        let thread_root = std::env::temp_dir().join(format!(
6844            "roder-provider-compaction-boundary-{}",
6845            uuid::Uuid::new_v4()
6846        ));
6847        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
6848            base_path: thread_root.clone(),
6849        }));
6850        let runtime = Arc::new(
6851            Runtime::new(
6852                builder.build().unwrap(),
6853                RuntimeConfig {
6854                    default_provider: PROVIDER_MOCK.to_string(),
6855                    default_model: "gpt-5.5".to_string(),
6856                    ..RuntimeConfig::default()
6857                },
6858            )
6859            .unwrap(),
6860        );
6861        let thread_id = runtime
6862            .create_thread(Some("Provider compaction boundary".to_string()))
6863            .await
6864            .unwrap()
6865            .thread_id;
6866        let old_turn = "old-turn".to_string();
6867        runtime
6868            .persist_turn_item(
6869                &thread_id,
6870                &old_turn,
6871                &TranscriptItem::UserMessage(UserMessage::text(
6872                    "old history that must not be replayed after provider compaction",
6873                )),
6874            )
6875            .await
6876            .unwrap();
6877        runtime
6878            .persist_turn_item(
6879                &thread_id,
6880                &old_turn,
6881                &TranscriptItem::AssistantMessage(AssistantMessage {
6882                    text: "old answer".to_string(),
6883                    phase: None,
6884                }),
6885            )
6886            .await
6887            .unwrap();
6888        runtime
6889            .persist_turn_item(
6890                &thread_id,
6891                &old_turn,
6892                &TranscriptItem::ProviderMetadata(serde_json::json!({
6893                    "output": [{
6894                        "id": "cmp_1",
6895                        "type": "compaction",
6896                        "encrypted_content": "opaque-state"
6897                    }]
6898                })),
6899            )
6900            .await
6901            .unwrap();
6902        runtime
6903            .persist_turn_item(
6904                &thread_id,
6905                &old_turn,
6906                &TranscriptItem::AssistantMessage(AssistantMessage {
6907                    text: "after compact".to_string(),
6908                    phase: None,
6909                }),
6910            )
6911            .await
6912            .unwrap();
6913
6914        let mut events = runtime.subscribe_events();
6915        runtime
6916            .start_turn(StartTurnRequest {
6917                thread_id: thread_id.clone(),
6918                message: "continue".to_string(),
6919                images: Vec::new(),
6920                provider_override: None,
6921                model_override: None,
6922                reasoning_override: None,
6923                workspace: test_workspace(),
6924                instructions: InstructionBundle::default(),
6925                developer_context: None,
6926                task_ledger_required: false,
6927                service_tier_override: None,
6928            })
6929            .await
6930            .unwrap();
6931        loop {
6932            let envelope = tokio::time::timeout(std::time::Duration::from_secs(5), events.recv())
6933                .await
6934                .unwrap()
6935                .unwrap();
6936            if envelope.thread_id.as_deref() == Some(&thread_id)
6937                && matches!(envelope.event, RoderEvent::TurnCompleted(_))
6938            {
6939                break;
6940            }
6941        }
6942
6943        let request = captured.lock().unwrap().clone().unwrap();
6944        let compaction_idx = request.transcript.iter().position(|item| {
6945            matches!(
6946                item,
6947                TranscriptItem::ProviderMetadata(metadata)
6948                    if crate::compaction::provider_metadata_has_compaction(metadata)
6949            )
6950        });
6951        assert!(
6952            compaction_idx.is_some(),
6953            "next provider request should include the provider compaction item: {:?}",
6954            request.transcript
6955        );
6956        // Skill/context injectors may prepend non-history items, but no prior-turn
6957        // conversation may appear before the latest provider compaction boundary.
6958        let history_before_boundary =
6959            request
6960                .transcript
6961                .iter()
6962                .take(compaction_idx.unwrap())
6963                .any(|item| match item {
6964                    TranscriptItem::AssistantMessage(_)
6965                    | TranscriptItem::ToolCall(_)
6966                    | TranscriptItem::ToolResult(_) => true,
6967                    TranscriptItem::UserMessage(message) => {
6968                        !message.text.contains("<skills>") && message.text != "continue"
6969                    }
6970                    _ => false,
6971                });
6972        assert!(
6973            !history_before_boundary,
6974            "pre-provider-compaction conversation must not be replayed: {:?}",
6975            request.transcript
6976        );
6977        assert!(
6978            !request.transcript.iter().any(|item| {
6979                matches!(
6980                    item,
6981                    TranscriptItem::UserMessage(message)
6982                        if message.text.contains("old history that must not be replayed")
6983                )
6984            }),
6985            "pre-provider-compaction history must not be replayed: {:?}",
6986            request.transcript
6987        );
6988        assert!(
6989            request.transcript.iter().any(|item| {
6990                matches!(
6991                    item,
6992                    TranscriptItem::UserMessage(message) if message.text == "continue"
6993                )
6994            }),
6995            "current continue prompt must be preserved: {:?}",
6996            request.transcript
6997        );
6998        assert_eq!(
6999            request.runtime.auto_compact_token_limit,
7000            Some(945_000),
7001            "gpt-5.5 should keep server-side auto compaction configured"
7002        );
7003
7004        let _ = std::fs::remove_dir_all(thread_root);
7005    }
7006
7007    #[test]
7008    fn server_side_compaction_is_only_enabled_for_supported_models() {
7009        let cfg = RuntimeConfig {
7010            auto_compact_token_limit: Some(123_456),
7011            ..RuntimeConfig::default()
7012        };
7013
7014        assert_eq!(server_side_compaction_threshold(&cfg, "mock"), None);
7015        assert_eq!(
7016            server_side_compaction_threshold(&cfg, "codex-auto-review"),
7017            None
7018        );
7019    }
7020
7021    #[test]
7022    fn reasoning_is_disabled_for_models_without_reasoning_support() {
7023        let cfg = RuntimeConfig {
7024            reasoning: Some(REASONING_HIGH.to_string()),
7025            ..RuntimeConfig::default()
7026        };
7027
7028        assert_eq!(
7029            effective_reasoning_for_model(&cfg, "claude-haiku-4-5-20251001"),
7030            REASONING_NONE
7031        );
7032        assert_eq!(
7033            reasoning_for_model(&cfg, "claude-haiku-4-5-20251001"),
7034            ReasoningConfig::default()
7035        );
7036    }
7037
7038    #[test]
7039    fn unsupported_configured_reasoning_falls_back_to_model_default() {
7040        let cfg = RuntimeConfig {
7041            reasoning: Some(REASONING_MINIMAL.to_string()),
7042            ..RuntimeConfig::default()
7043        };
7044
7045        assert_eq!(
7046            effective_reasoning_for_model(&cfg, "gpt-5.5"),
7047            REASONING_MEDIUM
7048        );
7049    }
7050
7051    #[test]
7052    fn unsupported_configured_gemini_reasoning_is_rejected() {
7053        let mut builder = ExtensionRegistryBuilder::new();
7054        builder.inference_engine(std::sync::Arc::new(FakeInferenceEngine));
7055
7056        let err = match Runtime::new(
7057            builder.build().unwrap(),
7058            RuntimeConfig {
7059                default_model: "gemini-3.5-flash".to_string(),
7060                reasoning: Some(REASONING_XHIGH.to_string()),
7061                ..RuntimeConfig::default()
7062            },
7063        ) {
7064            Ok(_) => panic!("expected unsupported Gemini reasoning to be rejected"),
7065            Err(err) => err,
7066        };
7067
7068        assert!(
7069            err.to_string()
7070                .contains("model gemini-3.5-flash does not support reasoning effort xhigh")
7071        );
7072    }
7073
7074    #[tokio::test]
7075    async fn selecting_none_for_non_reasoning_model_preserves_stored_preference() {
7076        let runtime = Runtime::new(
7077            Runtime::fake().unwrap().registry,
7078            RuntimeConfig {
7079                reasoning: Some(REASONING_HIGH.to_string()),
7080                ..RuntimeConfig::default()
7081            },
7082        )
7083        .unwrap();
7084
7085        let cfg = runtime
7086            .select_provider(
7087                roder_api::catalog::PROVIDER_MOCK.to_string(),
7088                Some("claude-haiku-4-5-20251001".to_string()),
7089                Some(REASONING_NONE.to_string()),
7090            )
7091            .await
7092            .unwrap();
7093
7094        assert_eq!(cfg.reasoning.as_deref(), Some(REASONING_HIGH));
7095        assert_eq!(runtime.effective_reasoning().await, REASONING_NONE);
7096    }
7097
7098    #[tokio::test]
7099    async fn selecting_none_for_model_that_supports_none_updates_preference() {
7100        let runtime = Runtime::new(
7101            Runtime::fake().unwrap().registry,
7102            RuntimeConfig {
7103                reasoning: Some(REASONING_HIGH.to_string()),
7104                ..RuntimeConfig::default()
7105            },
7106        )
7107        .unwrap();
7108
7109        let cfg = runtime
7110            .select_provider(
7111                roder_api::catalog::PROVIDER_MOCK.to_string(),
7112                Some("mock".to_string()),
7113                Some(REASONING_NONE.to_string()),
7114            )
7115            .await
7116            .unwrap();
7117
7118        assert_eq!(cfg.reasoning.as_deref(), Some(REASONING_NONE));
7119    }
7120
7121    #[test]
7122    fn parallel_tool_calls_default_on_with_model_override() {
7123        assert!(parallel_tool_calls_for_model(
7124            &RuntimeConfig::default(),
7125            "custom-model"
7126        ));
7127
7128        let cfg = RuntimeConfig {
7129            model_parallel_tool_calls: std::collections::HashMap::from([(
7130                "custom-model".to_string(),
7131                false,
7132            )]),
7133            ..RuntimeConfig::default()
7134        };
7135
7136        assert!(!parallel_tool_calls_for_model(&cfg, "custom-model"));
7137        assert!(parallel_tool_calls_for_model(&cfg, "other-model"));
7138    }
7139
7140    #[test]
7141    fn profile_parallel_tool_calls_applies_between_config_and_default() {
7142        let cfg = RuntimeConfig {
7143            model_profiles: std::collections::HashMap::from([(
7144                "gpt-5.5".to_string(),
7145                test_model_profile("gpt-5.5"),
7146            )]),
7147            ..RuntimeConfig::default()
7148        };
7149
7150        assert!(!parallel_tool_calls_for_model(&cfg, "gpt-5.5"));
7151
7152        let cfg = RuntimeConfig {
7153            model_parallel_tool_calls: std::collections::HashMap::from([(
7154                "gpt-5.5".to_string(),
7155                true,
7156            )]),
7157            ..cfg
7158        };
7159
7160        assert!(parallel_tool_calls_for_model(&cfg, "gpt-5.5"));
7161    }
7162
7163    struct CapturingEngine {
7164        request: Arc<StdMutex<Option<AgentInferenceRequest>>>,
7165    }
7166
7167    #[async_trait::async_trait]
7168    impl InferenceEngine for CapturingEngine {
7169        fn id(&self) -> String {
7170            roder_api::catalog::PROVIDER_MOCK.to_string()
7171        }
7172
7173        fn capabilities(&self) -> InferenceCapabilities {
7174            InferenceCapabilities::coding_agent_default()
7175        }
7176
7177        async fn list_models(
7178            &self,
7179            _ctx: InferenceProviderContext<'_>,
7180        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7181            Ok(roder_api::catalog::models_for_provider(
7182                roder_api::catalog::PROVIDER_MOCK,
7183                true,
7184            ))
7185        }
7186
7187        async fn stream_turn(
7188            &self,
7189            _ctx: InferenceTurnContext<'_>,
7190            request: AgentInferenceRequest,
7191        ) -> anyhow::Result<InferenceEventStream> {
7192            *self.request.lock().unwrap() = Some(request);
7193            Ok(Box::pin(stream::iter(vec![
7194                Ok(InferenceEvent::MessageDelta(MessageDelta {
7195                    text: "done".to_string(),
7196                    phase: None,
7197                })),
7198                Ok(InferenceEvent::Completed(CompletionMetadata {
7199                    stop_reason: Some("stop".to_string()),
7200                    provider_response_id: None,
7201                })),
7202            ])))
7203        }
7204    }
7205
7206    struct RoutingCaptureEngine {
7207        id: &'static str,
7208        models: Vec<ModelDescriptor>,
7209        requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
7210    }
7211
7212    #[async_trait::async_trait]
7213    impl InferenceEngine for RoutingCaptureEngine {
7214        fn id(&self) -> String {
7215            self.id.to_string()
7216        }
7217
7218        fn capabilities(&self) -> InferenceCapabilities {
7219            InferenceCapabilities::coding_agent_default()
7220        }
7221
7222        async fn list_models(
7223            &self,
7224            _ctx: InferenceProviderContext<'_>,
7225        ) -> anyhow::Result<Vec<ModelDescriptor>> {
7226            Ok(self.models.clone())
7227        }
7228
7229        async fn stream_turn(
7230            &self,
7231            _ctx: InferenceTurnContext<'_>,
7232            request: AgentInferenceRequest,
7233        ) -> anyhow::Result<InferenceEventStream> {
7234            self.requests.lock().unwrap().push(request);
7235            Ok(Box::pin(stream::iter(vec![
7236                Ok(InferenceEvent::MessageDelta(MessageDelta {
7237                    text: "routed".to_string(),
7238                    phase: None,
7239                })),
7240                Ok(InferenceEvent::Completed(CompletionMetadata {
7241                    stop_reason: Some("stop".to_string()),
7242                    provider_response_id: None,
7243                })),
7244            ])))
7245        }
7246    }
7247
7248    struct StaticRouter {
7249        id: &'static str,
7250        decision: InferenceRoutingDecision,
7251        contexts: Arc<StdMutex<Vec<InferenceRoutingContext>>>,
7252    }
7253
7254    #[async_trait::async_trait]
7255    impl InferenceRouter for StaticRouter {
7256        fn id(&self) -> String {
7257            self.id.to_string()
7258        }
7259
7260        async fn route(
7261            &self,
7262            context: InferenceRoutingContext,
7263        ) -> anyhow::Result<InferenceRoutingDecision> {
7264            self.contexts.lock().unwrap().push(context);
7265            Ok(self.decision.clone())
7266        }
7267    }
7268
7269    fn routing_test_model(id: &str, supported_reasoning: &[&str]) -> ModelDescriptor {
7270        ModelDescriptor {
7271            id: id.to_string(),
7272            name: id.to_string(),
7273            context_window: Some(128_000),
7274            default_reasoning: supported_reasoning
7275                .first()
7276                .map(|effort| (*effort).to_string()),
7277            supported_reasoning: supported_reasoning
7278                .iter()
7279                .map(|effort| ReasoningEffortDescriptor {
7280                    effort: (*effort).to_string(),
7281                    description: format!("{effort} reasoning"),
7282                })
7283                .collect(),
7284        }
7285    }
7286
7287    struct TaskLedgerCompletionGateEngine {
7288        calls: StdMutex<u32>,
7289        requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
7290    }
7291
7292    #[async_trait::async_trait]
7293    impl InferenceEngine for TaskLedgerCompletionGateEngine {
7294        fn id(&self) -> String {
7295            roder_api::catalog::PROVIDER_MOCK.to_string()
7296        }
7297
7298        fn capabilities(&self) -> InferenceCapabilities {
7299            InferenceCapabilities::coding_agent_default()
7300        }
7301
7302        async fn list_models(
7303            &self,
7304            _ctx: InferenceProviderContext<'_>,
7305        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7306            Ok(roder_api::catalog::models_for_provider(
7307                roder_api::catalog::PROVIDER_MOCK,
7308                true,
7309            ))
7310        }
7311
7312        async fn stream_turn(
7313            &self,
7314            _ctx: InferenceTurnContext<'_>,
7315            request: AgentInferenceRequest,
7316        ) -> anyhow::Result<InferenceEventStream> {
7317            self.requests.lock().unwrap().push(request);
7318            let mut calls = self.calls.lock().unwrap();
7319            *calls += 1;
7320            let events = match *calls {
7321                1 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7322                    id: "ledger-open".to_string(),
7323                    name: TASK_LEDGER_TOOL_NAME.to_string(),
7324                    arguments: serde_json::json!({
7325                        "tasks": [
7326                            {
7327                                "id": "inspect",
7328                                "content": "Inspect local assets",
7329                                "status": "completed",
7330                                "evidence": "listed workspace"
7331                            },
7332                            {
7333                                "id": "write",
7334                                "content": "Write /app/result.txt",
7335                                "status": "pending"
7336                            }
7337                        ],
7338                        "requireCompletionEvidence": true
7339                    })
7340                    .to_string(),
7341                }))],
7342                3 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7343                    id: "ledger-complete".to_string(),
7344                    name: TASK_LEDGER_TOOL_NAME.to_string(),
7345                    arguments: serde_json::json!({
7346                        "tasks": [
7347                            {
7348                                "id": "inspect",
7349                                "content": "Inspect local assets",
7350                                "status": "completed",
7351                                "evidence": "listed workspace"
7352                            },
7353                            {
7354                                "id": "write",
7355                                "content": "Write /app/result.txt",
7356                                "status": "completed",
7357                                "evidence": "wrote answer"
7358                            }
7359                        ],
7360                        "requireCompletionEvidence": true
7361                    })
7362                    .to_string(),
7363                }))],
7364                _ => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
7365                    text: "final".to_string(),
7366                    phase: None,
7367                }))],
7368            };
7369            Ok(Box::pin(stream::iter(events.into_iter().chain(
7370                std::iter::once(Ok(InferenceEvent::Completed(CompletionMetadata {
7371                    stop_reason: Some("stop".to_string()),
7372                    provider_response_id: None,
7373                }))),
7374            ))))
7375        }
7376    }
7377
7378    struct VerificationGateEngine {
7379        calls: StdMutex<u32>,
7380    }
7381
7382    #[async_trait::async_trait]
7383    impl InferenceEngine for VerificationGateEngine {
7384        fn id(&self) -> String {
7385            roder_api::catalog::PROVIDER_MOCK.to_string()
7386        }
7387
7388        fn capabilities(&self) -> InferenceCapabilities {
7389            InferenceCapabilities::coding_agent_default()
7390        }
7391
7392        async fn list_models(
7393            &self,
7394            _ctx: InferenceProviderContext<'_>,
7395        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7396            Ok(roder_api::catalog::models_for_provider(
7397                roder_api::catalog::PROVIDER_MOCK,
7398                true,
7399            ))
7400        }
7401
7402        async fn stream_turn(
7403            &self,
7404            _ctx: InferenceTurnContext<'_>,
7405            request: AgentInferenceRequest,
7406        ) -> anyhow::Result<InferenceEventStream> {
7407            let mut calls = self.calls.lock().unwrap();
7408            *calls += 1;
7409            let events = match *calls {
7410                1 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7411                    id: "write-1".to_string(),
7412                    name: "write_file".to_string(),
7413                    arguments: serde_json::json!({
7414                        "path": "src/lib.rs",
7415                        "content": "pub fn answer() -> u8 { 42 }\n"
7416                    })
7417                    .to_string(),
7418                }))],
7419                2 => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
7420                    text: "done too early".to_string(),
7421                    phase: None,
7422                }))],
7423                3 if request.transcript.iter().any(|item| {
7424                    matches!(
7425                        item,
7426                        TranscriptItem::UserMessage(message)
7427                            if message.text.contains("Verification gate blocked final completion")
7428                    )
7429                }) =>
7430                {
7431                    vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7432                        id: "verify-1".to_string(),
7433                        name: crate::verification_gate::VERIFICATION_TOOL_NAME.to_string(),
7434                        arguments: serde_json::json!({
7435                            "originalTask": "write code",
7436                            "changedFiles": ["src/lib.rs"],
7437                            "toolEvidence": ["write_file wrote src/lib.rs"],
7438                            "testsRun": ["cargo test -p roder-core verification_gate"],
7439                            "openGaps": [],
7440                            "status": "completed"
7441                        })
7442                        .to_string(),
7443                    }))]
7444                }
7445                _ => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
7446                    text: "verified final".to_string(),
7447                    phase: None,
7448                }))],
7449            };
7450            Ok(Box::pin(stream::iter(events.into_iter().chain(
7451                std::iter::once(Ok(InferenceEvent::Completed(CompletionMetadata {
7452                    stop_reason: Some("stop".to_string()),
7453                    provider_response_id: None,
7454                }))),
7455            ))))
7456        }
7457    }
7458
7459    struct SpeedPolicyEngine {
7460        calls: StdMutex<u32>,
7461        requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
7462    }
7463
7464    #[async_trait::async_trait]
7465    impl InferenceEngine for SpeedPolicyEngine {
7466        fn id(&self) -> String {
7467            roder_api::catalog::PROVIDER_MOCK.to_string()
7468        }
7469
7470        fn capabilities(&self) -> InferenceCapabilities {
7471            InferenceCapabilities::coding_agent_default()
7472        }
7473
7474        async fn list_models(
7475            &self,
7476            _ctx: InferenceProviderContext<'_>,
7477        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7478            Ok(roder_api::catalog::models_for_provider(
7479                roder_api::catalog::PROVIDER_MOCK,
7480                true,
7481            ))
7482        }
7483
7484        async fn stream_turn(
7485            &self,
7486            _ctx: InferenceTurnContext<'_>,
7487            request: AgentInferenceRequest,
7488        ) -> anyhow::Result<InferenceEventStream> {
7489            self.requests.lock().unwrap().push(request.clone());
7490            let mut calls = self.calls.lock().unwrap();
7491            *calls += 1;
7492            let events = match *calls {
7493                1 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7494                    id: "write-1".to_string(),
7495                    name: "write_file".to_string(),
7496                    arguments: serde_json::json!({
7497                        "path": "src/lib.rs",
7498                        "content": "pub fn answer() -> u8 { 42 }\n"
7499                    })
7500                    .to_string(),
7501                }))],
7502                3 if request.transcript.iter().any(|item| {
7503                    matches!(
7504                        item,
7505                        TranscriptItem::UserMessage(message)
7506                            if message.text.contains("Verification gate blocked final completion")
7507                    )
7508                }) =>
7509                {
7510                    vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7511                        id: "verify-1".to_string(),
7512                        name: crate::verification_gate::VERIFICATION_TOOL_NAME.to_string(),
7513                        arguments: serde_json::json!({
7514                            "originalTask": "write code",
7515                            "changedFiles": ["src/lib.rs"],
7516                            "toolEvidence": ["write_file wrote src/lib.rs"],
7517                            "testsRun": ["cargo test -p roder-core speed_policy"],
7518                            "openGaps": [],
7519                            "status": "completed"
7520                        })
7521                        .to_string(),
7522                    }))]
7523                }
7524                _ => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
7525                    text: "done".to_string(),
7526                    phase: None,
7527                }))],
7528            };
7529            Ok(Box::pin(stream::iter(events.into_iter().chain(
7530                std::iter::once(Ok(InferenceEvent::Completed(CompletionMetadata {
7531                    stop_reason: Some("stop".to_string()),
7532                    provider_response_id: None,
7533                }))),
7534            ))))
7535        }
7536    }
7537
7538    struct SwitchCaptureEngine {
7539        requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
7540    }
7541
7542    #[async_trait::async_trait]
7543    impl InferenceEngine for SwitchCaptureEngine {
7544        fn id(&self) -> String {
7545            roder_api::catalog::PROVIDER_MOCK.to_string()
7546        }
7547
7548        fn capabilities(&self) -> InferenceCapabilities {
7549            InferenceCapabilities::coding_agent_default()
7550        }
7551
7552        async fn list_models(
7553            &self,
7554            _ctx: InferenceProviderContext<'_>,
7555        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7556            Ok(roder_api::catalog::models_for_provider(
7557                roder_api::catalog::PROVIDER_MOCK,
7558                true,
7559            ))
7560        }
7561
7562        async fn stream_turn(
7563            &self,
7564            _ctx: InferenceTurnContext<'_>,
7565            request: AgentInferenceRequest,
7566        ) -> anyhow::Result<InferenceEventStream> {
7567            self.requests.lock().unwrap().push(request);
7568            Ok(Box::pin(stream::iter(vec![
7569                Ok(InferenceEvent::MessageDelta(MessageDelta {
7570                    text: "done".to_string(),
7571                    phase: None,
7572                })),
7573                Ok(InferenceEvent::Completed(CompletionMetadata {
7574                    stop_reason: Some("stop".to_string()),
7575                    provider_response_id: None,
7576                })),
7577            ])))
7578        }
7579    }
7580
7581    struct DeadlineEngine;
7582
7583    #[async_trait::async_trait]
7584    impl InferenceEngine for DeadlineEngine {
7585        fn id(&self) -> String {
7586            roder_api::catalog::PROVIDER_MOCK.to_string()
7587        }
7588
7589        fn capabilities(&self) -> InferenceCapabilities {
7590            InferenceCapabilities::coding_agent_default()
7591        }
7592
7593        async fn list_models(
7594            &self,
7595            _ctx: InferenceProviderContext<'_>,
7596        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7597            Ok(Vec::new())
7598        }
7599
7600        async fn stream_turn(
7601            &self,
7602            _ctx: InferenceTurnContext<'_>,
7603            _request: AgentInferenceRequest,
7604        ) -> anyhow::Result<InferenceEventStream> {
7605            Ok(Box::pin(stream::once(async {
7606                tokio::time::sleep(std::time::Duration::from_secs(60)).await;
7607                Ok(InferenceEvent::MessageDelta(MessageDelta {
7608                    text: "too late".to_string(),
7609                    phase: None,
7610                }))
7611            })))
7612        }
7613    }
7614
7615    struct WriteFileContributor;
7616
7617    impl ToolContributor for WriteFileContributor {
7618        fn id(&self) -> String {
7619            "test-write".to_string()
7620        }
7621
7622        fn contribute(&self, registry: &mut ToolRegistry) -> anyhow::Result<()> {
7623            registry.register(Arc::new(WriteFileTool))
7624        }
7625    }
7626
7627    struct WriteFileTool;
7628
7629    #[async_trait::async_trait]
7630    impl ToolExecutor for WriteFileTool {
7631        fn spec(&self) -> ToolSpec {
7632            ToolSpec {
7633                name: "write_file".to_string(),
7634                description: "Write a test file.".to_string(),
7635                parameters: serde_json::json!({
7636                    "type": "object",
7637                    "properties": {
7638                        "path": { "type": "string" },
7639                        "content": { "type": "string" }
7640                    },
7641                    "required": ["path", "content"],
7642                    "additionalProperties": false
7643                }),
7644            }
7645        }
7646
7647        async fn execute(
7648            &self,
7649            _ctx: ToolExecutionContext,
7650            call: ToolCall,
7651        ) -> anyhow::Result<ToolResult> {
7652            let path = call
7653                .arguments
7654                .get("path")
7655                .and_then(serde_json::Value::as_str)
7656                .unwrap_or("src/lib.rs");
7657            Ok(ToolResult {
7658                id: call.id,
7659                name: call.name,
7660                text: format!("wrote {path}"),
7661                data: serde_json::json!({ "path": path }),
7662                is_error: false,
7663            })
7664        }
7665    }
7666
7667    struct ProfileToolContributor;
7668
7669    impl ToolContributor for ProfileToolContributor {
7670        fn id(&self) -> String {
7671            "profile-tools".to_string()
7672        }
7673
7674        fn contribute(&self, registry: &mut ToolRegistry) -> anyhow::Result<()> {
7675            for name in ["apply_patch", "edit", "multi_edit", "write_file"] {
7676                registry.register(Arc::new(ProfileTool {
7677                    name: name.to_string(),
7678                }))?;
7679            }
7680            Ok(())
7681        }
7682    }
7683
7684    struct ProfileTool {
7685        name: String,
7686    }
7687
7688    #[async_trait::async_trait]
7689    impl ToolExecutor for ProfileTool {
7690        fn spec(&self) -> ToolSpec {
7691            ToolSpec {
7692                name: self.name.clone(),
7693                description: format!("{} test tool", self.name),
7694                parameters: serde_json::json!({
7695                    "type": "object",
7696                    "properties": {
7697                        "path": { "type": "string" },
7698                        "content": { "type": "string" }
7699                    },
7700                    "required": ["path", "content"],
7701                    "additionalProperties": false
7702                }),
7703            }
7704        }
7705
7706        async fn execute(
7707            &self,
7708            _ctx: ToolExecutionContext,
7709            call: ToolCall,
7710        ) -> anyhow::Result<ToolResult> {
7711            Ok(ToolResult {
7712                id: call.id,
7713                name: call.name,
7714                text: "ok".to_string(),
7715                data: serde_json::json!({}),
7716                is_error: false,
7717            })
7718        }
7719    }
7720
7721    fn test_model_profile(model: &str) -> ModelHarnessProfile {
7722        ModelHarnessProfile {
7723            model: model.to_string(),
7724            provider: roder_api::catalog::PROVIDER_OPENAI.to_string(),
7725            provider_family: ProviderFamily::OpenAi,
7726            edit_tool: Some(EDIT_TOOL_EDIT.to_string()),
7727            schema_policy: ModelSchemaPolicy::StandardRequiredFirst,
7728            instruction_overlay: ModelInstructionOverlay::IntuitiveContext,
7729            reasoning: ModelProfileReasoning {
7730                orientation: Some(REASONING_LOW.to_string()),
7731                execution: Some(REASONING_LOW.to_string()),
7732                verification: Some(REASONING_LOW.to_string()),
7733                recovery: Some(REASONING_LOW.to_string()),
7734            },
7735            parallel_tool_calls: Some(false),
7736            auto_compact_token_limit: Some(123_000),
7737        }
7738    }
7739
7740    async fn captured_profile_request(cfg: RuntimeConfig) -> AgentInferenceRequest {
7741        let captured = Arc::new(StdMutex::new(None));
7742        let mut builder = ExtensionRegistryBuilder::new();
7743        builder.inference_engine(Arc::new(CapturingEngine {
7744            request: captured.clone(),
7745        }));
7746        builder.tool_contributor(Arc::new(ProfileToolContributor));
7747        let runtime = Arc::new(Runtime::new(builder.build().unwrap(), cfg).unwrap());
7748        let mut rx = runtime.subscribe_events();
7749        let turn_id = runtime
7750            .start_turn(StartTurnRequest {
7751                thread_id: "thread-model-profile".to_string(),
7752                message: "use profile knobs".to_string(),
7753                images: Vec::new(),
7754                provider_override: None,
7755                model_override: None,
7756                reasoning_override: None,
7757                workspace: test_workspace(),
7758                instructions: InstructionBundle {
7759                    system: None,
7760                    developer: Some("base developer".to_string()),
7761                    developer_context: None,
7762                },
7763                developer_context: None,
7764                task_ledger_required: false,
7765                service_tier_override: None,
7766            })
7767            .await
7768            .unwrap();
7769
7770        tokio::time::timeout(std::time::Duration::from_secs(5), async {
7771            loop {
7772                let envelope = rx.recv().await.unwrap();
7773                if envelope.turn_id.as_deref() != Some(&turn_id) {
7774                    continue;
7775                }
7776                match envelope.event {
7777                    RoderEvent::TurnCompleted(_) => break,
7778                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7779                    _ => {}
7780                }
7781            }
7782        })
7783        .await
7784        .unwrap();
7785
7786        captured.lock().unwrap().clone().unwrap()
7787    }
7788
7789    struct ToolThenStopEngine {
7790        calls: StdMutex<u32>,
7791    }
7792
7793    #[async_trait::async_trait]
7794    impl InferenceEngine for ToolThenStopEngine {
7795        fn id(&self) -> String {
7796            roder_api::catalog::PROVIDER_MOCK.to_string()
7797        }
7798
7799        fn capabilities(&self) -> InferenceCapabilities {
7800            InferenceCapabilities::coding_agent_default()
7801        }
7802
7803        async fn list_models(
7804            &self,
7805            _ctx: InferenceProviderContext<'_>,
7806        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7807            Ok(roder_api::catalog::models_for_provider(
7808                roder_api::catalog::PROVIDER_MOCK,
7809                true,
7810            ))
7811        }
7812
7813        async fn stream_turn(
7814            &self,
7815            _ctx: InferenceTurnContext<'_>,
7816            _request: AgentInferenceRequest,
7817        ) -> anyhow::Result<InferenceEventStream> {
7818            let mut calls = self.calls.lock().unwrap();
7819            *calls += 1;
7820            let events = match *calls {
7821                1 => vec![
7822                    Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7823                        id: "write-1".to_string(),
7824                        name: "write_file".to_string(),
7825                        arguments: serde_json::json!({
7826                            "path": "src/lib.rs",
7827                            "content": "pub fn answer() -> u8 { 42 }\n"
7828                        })
7829                        .to_string(),
7830                    })),
7831                    Ok(InferenceEvent::Completed(CompletionMetadata {
7832                        stop_reason: Some("tool_use".to_string()),
7833                        provider_response_id: None,
7834                    })),
7835                ],
7836                _ => vec![
7837                    Ok(InferenceEvent::MessageDelta(MessageDelta {
7838                        text: "final".to_string(),
7839                        phase: None,
7840                    })),
7841                    Ok(InferenceEvent::Completed(CompletionMetadata {
7842                        stop_reason: Some("end_turn".to_string()),
7843                        provider_response_id: None,
7844                    })),
7845                ],
7846            };
7847            Ok(Box::pin(stream::iter(events)))
7848        }
7849    }
7850
7851    #[tokio::test]
7852    async fn turn_completed_reports_terminal_step_finish_reason() {
7853        let mut builder = ExtensionRegistryBuilder::new();
7854        builder.inference_engine(Arc::new(ToolThenStopEngine {
7855            calls: StdMutex::new(0),
7856        }));
7857        builder.tool_contributor(Arc::new(WriteFileContributor));
7858        let runtime = Arc::new(
7859            Runtime::new(
7860                builder.build().unwrap(),
7861                RuntimeConfig {
7862                    policy_mode: PolicyMode::Bypass,
7863                    agent_swarm_mode: false,
7864                    ultra_mode: false,
7865                    ..RuntimeConfig::default()
7866                },
7867            )
7868            .unwrap(),
7869        );
7870        let mut rx = runtime.subscribe_events();
7871        let turn_id = runtime
7872            .start_turn(StartTurnRequest {
7873                thread_id: "thread-finish-reason".to_string(),
7874                message: "write then finish".to_string(),
7875                images: Vec::new(),
7876                provider_override: None,
7877                model_override: None,
7878                reasoning_override: None,
7879                workspace: test_workspace(),
7880                instructions: InstructionBundle {
7881                    system: None,
7882                    developer: None,
7883                    developer_context: None,
7884                },
7885                developer_context: None,
7886                task_ledger_required: false,
7887                service_tier_override: None,
7888            })
7889            .await
7890            .unwrap();
7891
7892        let completed = tokio::time::timeout(std::time::Duration::from_secs(5), async {
7893            loop {
7894                let envelope = rx.recv().await.unwrap();
7895                if envelope.turn_id.as_deref() != Some(&turn_id) {
7896                    continue;
7897                }
7898                match envelope.event {
7899                    RoderEvent::TurnCompleted(event) => break event,
7900                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7901                    _ => {}
7902                }
7903            }
7904        })
7905        .await
7906        .unwrap();
7907
7908        // The mid-turn tool_use stop reason must not leak; the terminal
7909        // end_turn step decides the turn's finish reason.
7910        assert_eq!(completed.finish_reason.as_deref(), Some("stop"));
7911    }
7912
7913    #[tokio::test]
7914    async fn inference_router_selection_changes_request_model_and_records_event() {
7915        let default_requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7916        let routed_requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7917        let contexts = Arc::new(StdMutex::new(Vec::<InferenceRoutingContext>::new()));
7918        let selected = ModelSelection {
7919            provider: "routed-provider".to_string(),
7920            model: "routed-model".to_string(),
7921        };
7922        let default = ModelSelection {
7923            provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7924            model: "mock".to_string(),
7925        };
7926        let decision = InferenceRoutingDecision {
7927            reasoning: Some(ReasoningConfig {
7928                enabled: true,
7929                level: Some(REASONING_LOW.to_string()),
7930            }),
7931            confidence: Some(0.91),
7932            baseline: Some(default.clone()),
7933            matched_signals: vec![roder_api::inference_routing::InferenceRoutingSignal::new(
7934                "intent", "routine",
7935            )],
7936            ..InferenceRoutingDecision::selected("test-router", selected.clone(), "routine request")
7937        };
7938
7939        let mut builder = ExtensionRegistryBuilder::new();
7940        builder.inference_engine(Arc::new(RoutingCaptureEngine {
7941            id: roder_api::catalog::PROVIDER_MOCK,
7942            models: vec![routing_test_model("mock", &[REASONING_LOW])],
7943            requests: default_requests.clone(),
7944        }));
7945        builder.inference_engine(Arc::new(RoutingCaptureEngine {
7946            id: "routed-provider",
7947            models: vec![routing_test_model(
7948                "routed-model",
7949                &[REASONING_LOW, REASONING_MEDIUM],
7950            )],
7951            requests: routed_requests.clone(),
7952        }));
7953        builder.inference_router(Arc::new(StaticRouter {
7954            id: "test-router",
7955            decision,
7956            contexts: contexts.clone(),
7957        }));
7958        let thread_root =
7959            std::env::temp_dir().join(format!("roder-routing-auto-{}", uuid::Uuid::new_v4()));
7960        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
7961            base_path: thread_root.clone(),
7962        }));
7963        let runtime = Arc::new(
7964            Runtime::new(
7965                builder.build().unwrap(),
7966                RuntimeConfig {
7967                    default_provider: default.provider.clone(),
7968                    default_model: default.model.clone(),
7969                    ..RuntimeConfig::default()
7970                },
7971            )
7972            .unwrap(),
7973        );
7974        let thread_id = runtime
7975            .create_thread_with(CreateThreadRequest {
7976                title: Some("Routing auto".to_string()),
7977                workspace: test_workspace(),
7978                workspace_id: None,
7979                root_id: None,
7980                provider: Some(default.provider.clone()),
7981                model: Some(default.model.clone()),
7982                tool_allowlist: Vec::new(),
7983                developer_instructions: None,
7984                external_tools: Vec::new(),
7985                selection_mode: Some(ModelSelectionMode::auto(
7986                    "test-router:coding",
7987                    "test-router",
7988                    "Auto: Coding",
7989                    default.clone(),
7990                    Some("coding".to_string()),
7991                    None,
7992                )),
7993                runner: None,
7994            })
7995            .await
7996            .unwrap()
7997            .thread_id;
7998        let mut rx = runtime.subscribe_events();
7999        let turn_id = runtime
8000            .start_turn(StartTurnRequest {
8001                thread_id: thread_id.clone(),
8002                message: "small cleanup".to_string(),
8003                images: Vec::new(),
8004                provider_override: None,
8005                model_override: None,
8006                reasoning_override: None,
8007                workspace: test_workspace(),
8008                instructions: InstructionBundle::default(),
8009                developer_context: None,
8010                task_ledger_required: false,
8011                service_tier_override: None,
8012            })
8013            .await
8014            .unwrap();
8015
8016        let mut routing_event = None;
8017        let mut inference_started = None;
8018        tokio::time::timeout(std::time::Duration::from_secs(5), async {
8019            loop {
8020                let envelope = rx.recv().await.unwrap();
8021                if envelope.turn_id.as_deref() != Some(&turn_id) {
8022                    continue;
8023                }
8024                match envelope.event {
8025                    RoderEvent::InferenceRoutingDecision(event) => {
8026                        routing_event = Some(event);
8027                    }
8028                    RoderEvent::InferenceStarted(event) => {
8029                        inference_started = Some(event);
8030                    }
8031                    RoderEvent::TurnCompleted(_) => break,
8032                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8033                    _ => {}
8034                }
8035            }
8036        })
8037        .await
8038        .unwrap();
8039
8040        assert!(default_requests.lock().unwrap().is_empty());
8041        let routed_requests = routed_requests.lock().unwrap();
8042        assert_eq!(routed_requests.len(), 1);
8043        assert_eq!(routed_requests[0].model, selected);
8044        assert_eq!(
8045            routed_requests[0].reasoning.level.as_deref(),
8046            Some(REASONING_LOW)
8047        );
8048        assert_eq!(
8049            routed_requests[0].metadata["inferenceRouting"]["outcome"],
8050            "selected"
8051        );
8052
8053        let routing_event = routing_event.expect("routing decision event");
8054        assert_eq!(routing_event.default_selection, default);
8055        assert_eq!(routing_event.selected_selection, selected);
8056        assert_eq!(
8057            routing_event.decision.outcome,
8058            InferenceRoutingOutcome::Selected
8059        );
8060        assert_eq!(
8061            inference_started.expect("inference started event").model,
8062            selected
8063        );
8064
8065        let contexts = contexts.lock().unwrap();
8066        assert_eq!(contexts.len(), 1);
8067        assert_eq!(contexts[0].default_selection, default);
8068        assert_eq!(contexts[0].candidates.len(), 2);
8069        assert!(
8070            contexts[0]
8071                .signals
8072                .iter()
8073                .any(|signal| signal.key == "profile" && signal.value == "coding")
8074        );
8075        let _ = std::fs::remove_dir_all(thread_root);
8076    }
8077
8078    #[tokio::test]
8079    async fn inference_router_is_bypassed_for_explicit_selection() {
8080        let requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
8081        let contexts = Arc::new(StdMutex::new(Vec::<InferenceRoutingContext>::new()));
8082        let selected = ModelSelection {
8083            provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8084            model: "mock".to_string(),
8085        };
8086
8087        let mut builder = ExtensionRegistryBuilder::new();
8088        builder.inference_engine(Arc::new(RoutingCaptureEngine {
8089            id: roder_api::catalog::PROVIDER_MOCK,
8090            models: vec![routing_test_model("mock", &[REASONING_LOW])],
8091            requests: requests.clone(),
8092        }));
8093        builder.inference_router(Arc::new(StaticRouter {
8094            id: "test-router",
8095            decision: InferenceRoutingDecision::selected(
8096                "test-router",
8097                ModelSelection {
8098                    provider: "missing".to_string(),
8099                    model: "missing".to_string(),
8100                },
8101                "would route if called",
8102            ),
8103            contexts: contexts.clone(),
8104        }));
8105        let thread_root =
8106            std::env::temp_dir().join(format!("roder-routing-explicit-{}", uuid::Uuid::new_v4()));
8107        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8108            base_path: thread_root.clone(),
8109        }));
8110        let runtime = Arc::new(
8111            Runtime::new(
8112                builder.build().unwrap(),
8113                RuntimeConfig {
8114                    default_provider: selected.provider.clone(),
8115                    default_model: selected.model.clone(),
8116                    inference_router: RuntimeInferenceRouterConfig {
8117                        enabled: true,
8118                        router_id: Some("test-router".to_string()),
8119                    },
8120                    ..RuntimeConfig::default()
8121                },
8122            )
8123            .unwrap(),
8124        );
8125        let thread_id = runtime
8126            .create_thread_with(CreateThreadRequest {
8127                title: Some("Routing explicit".to_string()),
8128                workspace: test_workspace(),
8129                workspace_id: None,
8130                root_id: None,
8131                provider: Some(selected.provider.clone()),
8132                model: Some(selected.model.clone()),
8133                tool_allowlist: Vec::new(),
8134                developer_instructions: None,
8135                external_tools: Vec::new(),
8136                selection_mode: Some(ModelSelectionMode::auto(
8137                    "test-router:default",
8138                    "test-router",
8139                    "Auto",
8140                    selected.clone(),
8141                    None,
8142                    None,
8143                )),
8144                runner: None,
8145            })
8146            .await
8147            .unwrap()
8148            .thread_id;
8149        let mut rx = runtime.subscribe_events();
8150        let turn_id = runtime
8151            .start_turn(StartTurnRequest {
8152                thread_id,
8153                message: "use explicit selection".to_string(),
8154                images: Vec::new(),
8155                provider_override: Some(selected.provider.clone()),
8156                model_override: Some(selected.model.clone()),
8157                reasoning_override: None,
8158                workspace: test_workspace(),
8159                instructions: InstructionBundle::default(),
8160                developer_context: None,
8161                task_ledger_required: false,
8162                service_tier_override: None,
8163            })
8164            .await
8165            .unwrap();
8166
8167        let mut saw_routing_event = false;
8168        tokio::time::timeout(std::time::Duration::from_secs(5), async {
8169            loop {
8170                let envelope = rx.recv().await.unwrap();
8171                if envelope.turn_id.as_deref() != Some(&turn_id) {
8172                    continue;
8173                }
8174                match envelope.event {
8175                    RoderEvent::InferenceRoutingDecision(_) => {
8176                        saw_routing_event = true;
8177                    }
8178                    RoderEvent::TurnCompleted(_) => break,
8179                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8180                    _ => {}
8181                }
8182            }
8183        })
8184        .await
8185        .unwrap();
8186
8187        assert!(!saw_routing_event);
8188        assert!(contexts.lock().unwrap().is_empty());
8189        let requests = requests.lock().unwrap();
8190        assert_eq!(requests.len(), 1);
8191        assert_eq!(requests[0].model, selected);
8192        let _ = std::fs::remove_dir_all(thread_root);
8193    }
8194
8195    #[tokio::test]
8196    async fn inference_router_is_bypassed_for_manual_selection_mode() {
8197        let requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
8198        let contexts = Arc::new(StdMutex::new(Vec::<InferenceRoutingContext>::new()));
8199        let selected = ModelSelection {
8200            provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8201            model: "mock".to_string(),
8202        };
8203
8204        let mut builder = ExtensionRegistryBuilder::new();
8205        builder.inference_engine(Arc::new(RoutingCaptureEngine {
8206            id: roder_api::catalog::PROVIDER_MOCK,
8207            models: vec![routing_test_model("mock", &[REASONING_LOW])],
8208            requests: requests.clone(),
8209        }));
8210        builder.inference_router(Arc::new(StaticRouter {
8211            id: "test-router",
8212            decision: InferenceRoutingDecision::selected(
8213                "test-router",
8214                ModelSelection {
8215                    provider: "missing".to_string(),
8216                    model: "missing".to_string(),
8217                },
8218                "would route if called",
8219            ),
8220            contexts: contexts.clone(),
8221        }));
8222        let thread_root =
8223            std::env::temp_dir().join(format!("roder-routing-manual-{}", uuid::Uuid::new_v4()));
8224        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8225            base_path: thread_root.clone(),
8226        }));
8227        let runtime = Arc::new(
8228            Runtime::new(
8229                builder.build().unwrap(),
8230                RuntimeConfig {
8231                    default_provider: selected.provider.clone(),
8232                    default_model: selected.model.clone(),
8233                    inference_router: RuntimeInferenceRouterConfig {
8234                        enabled: true,
8235                        router_id: Some("test-router".to_string()),
8236                    },
8237                    ..RuntimeConfig::default()
8238                },
8239            )
8240            .unwrap(),
8241        );
8242        let thread_id = runtime
8243            .create_thread_with(CreateThreadRequest {
8244                title: Some("Routing manual".to_string()),
8245                workspace: test_workspace(),
8246                workspace_id: None,
8247                root_id: None,
8248                provider: Some(selected.provider.clone()),
8249                model: Some(selected.model.clone()),
8250                tool_allowlist: Vec::new(),
8251                developer_instructions: None,
8252                external_tools: Vec::new(),
8253                selection_mode: Some(ModelSelectionMode::manual(
8254                    selected.provider.clone(),
8255                    selected.model.clone(),
8256                    None,
8257                )),
8258                runner: None,
8259            })
8260            .await
8261            .unwrap()
8262            .thread_id;
8263        let mut rx = runtime.subscribe_events();
8264        let turn_id = runtime
8265            .start_turn(StartTurnRequest {
8266                thread_id,
8267                message: "use selected manual model".to_string(),
8268                images: Vec::new(),
8269                provider_override: None,
8270                model_override: None,
8271                reasoning_override: None,
8272                workspace: test_workspace(),
8273                instructions: InstructionBundle::default(),
8274                developer_context: None,
8275                task_ledger_required: false,
8276                service_tier_override: None,
8277            })
8278            .await
8279            .unwrap();
8280
8281        let mut saw_routing_event = false;
8282        tokio::time::timeout(std::time::Duration::from_secs(5), async {
8283            loop {
8284                let envelope = rx.recv().await.unwrap();
8285                if envelope.turn_id.as_deref() != Some(&turn_id) {
8286                    continue;
8287                }
8288                match envelope.event {
8289                    RoderEvent::InferenceRoutingDecision(_) => {
8290                        saw_routing_event = true;
8291                    }
8292                    RoderEvent::TurnCompleted(_) => break,
8293                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8294                    _ => {}
8295                }
8296            }
8297        })
8298        .await
8299        .unwrap();
8300
8301        assert!(!saw_routing_event);
8302        assert!(contexts.lock().unwrap().is_empty());
8303        let requests = requests.lock().unwrap();
8304        assert_eq!(requests.len(), 1);
8305        assert_eq!(requests[0].model, selected);
8306        let _ = std::fs::remove_dir_all(thread_root);
8307    }
8308
8309    #[test]
8310    fn enabled_inference_router_requires_registered_router() {
8311        let mut builder = ExtensionRegistryBuilder::new();
8312        builder.inference_engine(Arc::new(FakeInferenceEngine));
8313
8314        let err = match Runtime::new(
8315            builder.build().unwrap(),
8316            RuntimeConfig {
8317                inference_router: RuntimeInferenceRouterConfig {
8318                    enabled: true,
8319                    router_id: Some("missing-router".to_string()),
8320                },
8321                ..RuntimeConfig::default()
8322            },
8323        ) {
8324            Ok(_) => panic!("runtime should reject unknown inference router"),
8325            Err(err) => err,
8326        };
8327
8328        assert!(
8329            err.to_string()
8330                .contains("inference router \"missing-router\" is not registered")
8331        );
8332    }
8333
8334    #[tokio::test]
8335    async fn model_profile_routes_request_knobs_to_next_inference() {
8336        let request = captured_profile_request(RuntimeConfig {
8337            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8338            default_model: "gpt-5.5".to_string(),
8339            model_profiles: std::collections::HashMap::from([(
8340                "gpt-5.5".to_string(),
8341                test_model_profile("gpt-5.5"),
8342            )]),
8343            ..RuntimeConfig::default()
8344        })
8345        .await;
8346
8347        let tool_names = request
8348            .tools
8349            .iter()
8350            .map(|tool| tool.name.as_str())
8351            .collect::<Vec<_>>();
8352        assert!(tool_names.contains(&"apply_patch"));
8353        assert!(tool_names.contains(&"edit"));
8354        assert!(tool_names.contains(&"multi_edit"));
8355        assert!(tool_names.contains(&"write_file"));
8356        assert_eq!(request.reasoning.level.as_deref(), Some(REASONING_LOW));
8357        assert_eq!(request.runtime.parallel_tool_calls, Some(false));
8358        assert_eq!(request.runtime.auto_compact_token_limit, Some(123_000));
8359        assert!(
8360            request
8361                .instructions
8362                .developer
8363                .as_deref()
8364                .unwrap_or_default()
8365                .contains("Use the provided context as the current working set")
8366        );
8367        assert_eq!(
8368            request
8369                .metadata
8370                .pointer("/modelProfile/schemaPolicy")
8371                .and_then(serde_json::Value::as_str),
8372            Some("standard_required_first")
8373        );
8374    }
8375
8376    #[tokio::test]
8377    async fn ultra_reasoning_reaches_transport_state_and_enables_proactive_delegation() {
8378        let request = captured_profile_request(RuntimeConfig {
8379            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8380            default_model: "gpt-5.6-sol".to_string(),
8381            reasoning: Some(REASONING_ULTRA.to_string()),
8382            ..RuntimeConfig::default()
8383        })
8384        .await;
8385
8386        assert_eq!(request.reasoning.level.as_deref(), Some(REASONING_ULTRA));
8387        let developer = request
8388            .instructions
8389            .developer
8390            .as_deref()
8391            .expect("ultra developer instructions");
8392        assert!(developer.contains("Proactive multi-agent delegation is active"));
8393    }
8394
8395    #[tokio::test]
8396    async fn lower_sol_effort_requires_explicit_multi_agent_request() {
8397        let request = captured_profile_request(RuntimeConfig {
8398            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8399            default_model: "gpt-5.6-sol".to_string(),
8400            reasoning: Some(roder_api::catalog::REASONING_MEDIUM.to_string()),
8401            ..RuntimeConfig::default()
8402        })
8403        .await;
8404
8405        let developer = request
8406            .instructions
8407            .developer
8408            .as_deref()
8409            .expect("sol developer instructions");
8410        assert!(developer.contains("Do not spawn sub-agents unless"));
8411        assert!(!developer.contains("Proactive multi-agent delegation is active"));
8412    }
8413
8414    #[tokio::test]
8415    async fn luna_does_not_receive_codex_v2_multi_agent_policy() {
8416        let request = captured_profile_request(RuntimeConfig {
8417            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8418            default_model: "gpt-5.6-luna".to_string(),
8419            reasoning: Some(roder_api::catalog::REASONING_MAX.to_string()),
8420            ..RuntimeConfig::default()
8421        })
8422        .await;
8423
8424        let developer = request
8425            .instructions
8426            .developer
8427            .as_deref()
8428            .unwrap_or_default();
8429        assert!(!developer.contains("Do not spawn sub-agents unless"));
8430        assert!(!developer.contains("Proactive multi-agent delegation is active"));
8431    }
8432
8433    #[tokio::test]
8434    async fn ultra_mode_enables_proactive_multi_agent_for_any_model() {
8435        // Luna has no Ultra effort, but ultra mode still injects proactive policy.
8436        let request = captured_profile_request(RuntimeConfig {
8437            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8438            default_model: "gpt-5.6-luna".to_string(),
8439            reasoning: Some(roder_api::catalog::REASONING_MAX.to_string()),
8440            ultra_mode: true,
8441            ..RuntimeConfig::default()
8442        })
8443        .await;
8444
8445        let developer = request
8446            .instructions
8447            .developer
8448            .as_deref()
8449            .expect("ultra mode developer instructions");
8450        assert!(developer.contains("Proactive multi-agent delegation is active"));
8451        assert!(developer.contains("spawn_agent"));
8452    }
8453
8454    #[tokio::test]
8455    async fn ultra_mode_overrides_explicit_request_only_on_sol() {
8456        // Medium Sol effort is normally explicit-request-only; ultra mode flips
8457        // it to proactive without requiring Ultra reasoning effort.
8458        let request = captured_profile_request(RuntimeConfig {
8459            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8460            default_model: "gpt-5.6-sol".to_string(),
8461            reasoning: Some(roder_api::catalog::REASONING_MEDIUM.to_string()),
8462            ultra_mode: true,
8463            ..RuntimeConfig::default()
8464        })
8465        .await;
8466
8467        let developer = request
8468            .instructions
8469            .developer
8470            .as_deref()
8471            .expect("ultra mode sol developer instructions");
8472        assert!(developer.contains("Proactive multi-agent delegation is active"));
8473        assert!(!developer.contains("Do not spawn sub-agents unless"));
8474    }
8475
8476    #[tokio::test]
8477    async fn turn_developer_context_reaches_inference_and_does_not_persist() {
8478        let captured = Arc::new(StdMutex::new(None));
8479        let mut builder = ExtensionRegistryBuilder::new();
8480        builder.inference_engine(Arc::new(CapturingEngine {
8481            request: captured.clone(),
8482        }));
8483        let runtime = Arc::new(
8484            Runtime::new(
8485                builder.build().unwrap(),
8486                RuntimeConfig {
8487                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8488                    default_model: "gpt-5.5".to_string(),
8489                    ..RuntimeConfig::default()
8490                },
8491            )
8492            .unwrap(),
8493        );
8494
8495        async fn run_turn(runtime: &Arc<Runtime>, developer_context: Option<String>) {
8496            let mut rx = runtime.subscribe_events();
8497            let turn_id = runtime
8498                .start_turn(StartTurnRequest {
8499                    thread_id: "thread-turn-context".to_string(),
8500                    message: "hello".to_string(),
8501                    images: Vec::new(),
8502                    provider_override: None,
8503                    model_override: None,
8504                    reasoning_override: None,
8505                    workspace: test_workspace(),
8506                    instructions: InstructionBundle::default(),
8507                    developer_context,
8508                    task_ledger_required: false,
8509                    service_tier_override: None,
8510                })
8511                .await
8512                .unwrap();
8513            tokio::time::timeout(std::time::Duration::from_secs(5), async {
8514                loop {
8515                    let envelope = rx.recv().await.unwrap();
8516                    if envelope.turn_id.as_deref() != Some(&turn_id) {
8517                        continue;
8518                    }
8519                    match envelope.event {
8520                        RoderEvent::TurnCompleted(_) => break,
8521                        RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8522                        _ => {}
8523                    }
8524                }
8525            })
8526            .await
8527            .unwrap();
8528        }
8529
8530        run_turn(
8531            &runtime,
8532            Some("Connected accounts: example-service.".to_string()),
8533        )
8534        .await;
8535        let request = captured.lock().unwrap().clone().unwrap();
8536        assert_eq!(
8537            request.instructions.developer_context.as_deref(),
8538            Some("Connected accounts: example-service.")
8539        );
8540
8541        // The context is per-turn only: the next turn on the same thread
8542        // without a developerContext must not see the previous one.
8543        run_turn(&runtime, None).await;
8544        let request = captured.lock().unwrap().clone().unwrap();
8545        assert_eq!(request.instructions.developer_context, None);
8546    }
8547
8548    #[tokio::test]
8549    async fn tool_search_overrides_route_to_next_inference_request() {
8550        let request = captured_profile_request(RuntimeConfig {
8551            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8552            default_model: "gpt-5.4".to_string(),
8553            tool_search: ToolSearchConfig {
8554                mode: roder_api::inference::ToolSearchMode::Auto,
8555                max_catalog_items: Some(100),
8556                ..ToolSearchConfig::default()
8557            },
8558            provider_tool_search: std::collections::HashMap::from([(
8559                roder_api::catalog::PROVIDER_MOCK.to_string(),
8560                roder_api::inference::ToolSearchConfigOverlay {
8561                    include_skills: Some(false),
8562                    provider_variant: Some(roder_api::inference::ToolSearchProviderVariant::Regex),
8563                    ..Default::default()
8564                },
8565            )]),
8566            model_tool_search: std::collections::HashMap::from([(
8567                "gpt-5.4".to_string(),
8568                roder_api::inference::ToolSearchConfigOverlay {
8569                    mode: Some(roder_api::inference::ToolSearchMode::ProviderNative),
8570                    max_catalog_items: Some(25),
8571                    provider_variant: Some(roder_api::inference::ToolSearchProviderVariant::Bm25),
8572                    ..Default::default()
8573                },
8574            )]),
8575            ..RuntimeConfig::default()
8576        })
8577        .await;
8578
8579        assert_eq!(
8580            request.runtime.tool_search.mode,
8581            roder_api::inference::ToolSearchMode::ProviderNative
8582        );
8583        assert_eq!(request.runtime.tool_search.max_catalog_items, Some(25));
8584        assert_eq!(
8585            request.runtime.tool_search.provider_variant,
8586            roder_api::inference::ToolSearchProviderVariant::Bm25
8587        );
8588    }
8589
8590    #[tokio::test]
8591    async fn context_entrypoint_hints_use_turn_workspace() {
8592        let process_workspace = runtime_test_workspace("entrypoint-process");
8593        let thread_workspace = runtime_test_workspace("entrypoint-thread");
8594        std::fs::create_dir_all(process_workspace.join("src")).unwrap();
8595        std::fs::create_dir_all(thread_workspace.join("src")).unwrap();
8596        std::fs::write(
8597            process_workspace.join("src/sidebar-thread-groups.ts"),
8598            "export const desktopLeak = true;\n",
8599        )
8600        .unwrap();
8601        std::fs::write(
8602            thread_workspace.join("src/voice-plan-feedback.ts"),
8603            "export const voicePlanFeedback = true;\n",
8604        )
8605        .unwrap();
8606
8607        let captured = Arc::new(StdMutex::new(None));
8608        let mut builder = ExtensionRegistryBuilder::new();
8609        builder.inference_engine(Arc::new(CapturingEngine {
8610            request: captured.clone(),
8611        }));
8612        builder.context_planner(Arc::new(roder_context::EntrypointContextPlanner::new(
8613            process_workspace.clone(),
8614        )));
8615        let runtime = Arc::new(
8616            Runtime::new(
8617                builder.build().unwrap(),
8618                RuntimeConfig {
8619                    workspace: Some(process_workspace.display().to_string()),
8620                    ..RuntimeConfig::default()
8621                },
8622            )
8623            .unwrap(),
8624        );
8625        let mut rx = runtime.subscribe_events();
8626
8627        let turn_id = runtime
8628            .start_turn(StartTurnRequest {
8629                thread_id: "thread-workspace-entrypoint".to_string(),
8630                message: "investigate voice plan feedback".to_string(),
8631                images: Vec::new(),
8632                provider_override: None,
8633                model_override: None,
8634                reasoning_override: None,
8635                workspace: thread_workspace.display().to_string(),
8636                instructions: crate::instructions::default_instructions(),
8637                developer_context: None,
8638                task_ledger_required: false,
8639                service_tier_override: None,
8640            })
8641            .await
8642            .unwrap();
8643
8644        tokio::time::timeout(std::time::Duration::from_secs(5), async {
8645            loop {
8646                let envelope = rx.recv().await.unwrap();
8647                if envelope.turn_id.as_deref() != Some(&turn_id) {
8648                    continue;
8649                }
8650                match envelope.event {
8651                    RoderEvent::TurnCompleted(_) => break,
8652                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8653                    _ => {}
8654                }
8655            }
8656        })
8657        .await
8658        .unwrap();
8659
8660        let request = captured.lock().unwrap().clone().expect("captured request");
8661        let transcript_text = request
8662            .transcript
8663            .iter()
8664            .map(|item| match item {
8665                TranscriptItem::UserMessage(message) => message.text.as_str(),
8666                _ => "",
8667            })
8668            .collect::<Vec<_>>()
8669            .join("\n");
8670        assert!(transcript_text.contains("src/voice-plan-feedback.ts"));
8671        assert!(!transcript_text.contains("src/sidebar-thread-groups.ts"));
8672
8673        let _ = std::fs::remove_dir_all(process_workspace);
8674        let _ = std::fs::remove_dir_all(thread_workspace);
8675    }
8676
8677    fn runtime_test_workspace(name: &str) -> std::path::PathBuf {
8678        let path =
8679            std::env::temp_dir().join(format!("roder-runtime-{name}-{}", uuid::Uuid::new_v4()));
8680        let _ = std::fs::remove_dir_all(&path);
8681        std::fs::create_dir_all(&path).unwrap();
8682        path
8683    }
8684
8685    #[tokio::test]
8686    async fn model_profile_user_model_knobs_override_profile_defaults() {
8687        let request = captured_profile_request(RuntimeConfig {
8688            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8689            default_model: "gpt-5.5".to_string(),
8690            reasoning: Some(REASONING_HIGH.to_string()),
8691            auto_compact_token_limit: Some(999),
8692            model_edit_tools: std::collections::HashMap::from([(
8693                "gpt-5.5".to_string(),
8694                EDIT_TOOL_PATCH.to_string(),
8695            )]),
8696            model_parallel_tool_calls: std::collections::HashMap::from([(
8697                "gpt-5.5".to_string(),
8698                true,
8699            )]),
8700            model_profiles: std::collections::HashMap::from([(
8701                "gpt-5.5".to_string(),
8702                test_model_profile("gpt-5.5"),
8703            )]),
8704            ..RuntimeConfig::default()
8705        })
8706        .await;
8707
8708        let tool_names = request
8709            .tools
8710            .iter()
8711            .map(|tool| tool.name.as_str())
8712            .collect::<Vec<_>>();
8713        assert!(tool_names.contains(&"apply_patch"));
8714        assert!(!tool_names.contains(&"edit"));
8715        assert_eq!(request.reasoning.level.as_deref(), Some(REASONING_HIGH));
8716        assert_eq!(request.runtime.parallel_tool_calls, Some(true));
8717        assert_eq!(request.runtime.auto_compact_token_limit, Some(999));
8718    }
8719
8720    #[tokio::test]
8721    async fn runtime_tool_allowlist_filters_advertised_tools() {
8722        let request = captured_profile_request(RuntimeConfig {
8723            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8724            default_model: "gpt-5.5".to_string(),
8725            tool_allowlist: vec!["edit".to_string()],
8726            model_profiles: std::collections::HashMap::from([(
8727                "gpt-5.5".to_string(),
8728                test_model_profile("gpt-5.5"),
8729            )]),
8730            ..RuntimeConfig::default()
8731        })
8732        .await;
8733
8734        let tool_names = request
8735            .tools
8736            .iter()
8737            .map(|tool| tool.name.as_str())
8738            .collect::<Vec<_>>();
8739        assert_eq!(tool_names, vec!["edit"]);
8740    }
8741
8742    /// Builds a store-backed runtime turn for a thread created with the given per-thread
8743    /// overrides and returns the captured inference request.
8744    async fn captured_thread_override_request(
8745        runtime: &Arc<Runtime>,
8746        requests: &Arc<StdMutex<Vec<AgentInferenceRequest>>>,
8747        tool_allowlist: Vec<String>,
8748        developer_instructions: Option<String>,
8749        external_tools: Vec<ToolSpec>,
8750    ) -> AgentInferenceRequest {
8751        let thread_id = runtime
8752            .create_thread_with(CreateThreadRequest {
8753                title: Some("Thread overrides".to_string()),
8754                workspace: test_workspace(),
8755                workspace_id: None,
8756                root_id: None,
8757                provider: None,
8758                model: None,
8759                selection_mode: None,
8760                tool_allowlist,
8761                developer_instructions,
8762                external_tools,
8763                runner: None,
8764            })
8765            .await
8766            .unwrap()
8767            .thread_id;
8768        let mut rx = runtime.subscribe_events();
8769        let turn_id = runtime
8770            .start_turn(StartTurnRequest {
8771                thread_id,
8772                message: "hello".to_string(),
8773                images: Vec::new(),
8774                provider_override: None,
8775                model_override: None,
8776                reasoning_override: None,
8777                workspace: test_workspace(),
8778                instructions: crate::default_instructions(),
8779                developer_context: None,
8780                task_ledger_required: false,
8781                service_tier_override: None,
8782            })
8783            .await
8784            .unwrap();
8785        tokio::time::timeout(std::time::Duration::from_secs(5), async {
8786            loop {
8787                let envelope = rx.recv().await.unwrap();
8788                if envelope.turn_id.as_deref() != Some(&turn_id) {
8789                    continue;
8790                }
8791                match envelope.event {
8792                    RoderEvent::TurnCompleted(_) => break,
8793                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8794                    _ => {}
8795                }
8796            }
8797        })
8798        .await
8799        .unwrap();
8800        requests.lock().unwrap().pop().expect("captured request")
8801    }
8802
8803    #[tokio::test]
8804    async fn thread_tool_allowlist_filters_only_that_thread() {
8805        let requests = Arc::new(StdMutex::new(Vec::new()));
8806        let thread_root =
8807            std::env::temp_dir().join(format!("roder-thread-allowlist-{}", uuid::Uuid::new_v4()));
8808        let mut builder = ExtensionRegistryBuilder::new();
8809        builder.inference_engine(Arc::new(SwitchCaptureEngine {
8810            requests: requests.clone(),
8811        }));
8812        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8813            base_path: thread_root.clone(),
8814        }));
8815        builder.tool_contributor(Arc::new(ProfileToolContributor));
8816        let runtime = Arc::new(
8817            Runtime::new(
8818                builder.build().unwrap(),
8819                RuntimeConfig {
8820                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8821                    default_model: "gpt-5.5".to_string(),
8822                    model_profiles: std::collections::HashMap::from([(
8823                        "gpt-5.5".to_string(),
8824                        test_model_profile("gpt-5.5"),
8825                    )]),
8826                    ..RuntimeConfig::default()
8827                },
8828            )
8829            .unwrap(),
8830        );
8831
8832        let allowlisted = captured_thread_override_request(
8833            &runtime,
8834            &requests,
8835            vec!["edit".to_string()],
8836            None,
8837            Vec::new(),
8838        )
8839        .await;
8840        let unrestricted =
8841            captured_thread_override_request(&runtime, &requests, Vec::new(), None, Vec::new())
8842                .await;
8843
8844        let allowlisted_names = allowlisted
8845            .tools
8846            .iter()
8847            .map(|tool| tool.name.as_str())
8848            .collect::<Vec<_>>();
8849        assert_eq!(allowlisted_names, vec!["edit"]);
8850        let unrestricted_names = unrestricted
8851            .tools
8852            .iter()
8853            .map(|tool| tool.name.as_str())
8854            .collect::<Vec<_>>();
8855        assert!(unrestricted_names.contains(&"edit"));
8856        assert!(unrestricted_names.len() > 1);
8857
8858        let _ = std::fs::remove_dir_all(thread_root);
8859    }
8860
8861    /// Builds a store-backed runtime whose `RuntimeConfig.tool_allowlist` is `["edit"]`.
8862    fn runtime_with_edit_allowlist(
8863        requests: &Arc<StdMutex<Vec<AgentInferenceRequest>>>,
8864        thread_root: &std::path::Path,
8865    ) -> Arc<Runtime> {
8866        let mut builder = ExtensionRegistryBuilder::new();
8867        builder.inference_engine(Arc::new(SwitchCaptureEngine {
8868            requests: requests.clone(),
8869        }));
8870        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8871            base_path: thread_root.to_path_buf(),
8872        }));
8873        builder.tool_contributor(Arc::new(ProfileToolContributor));
8874        Arc::new(
8875            Runtime::new(
8876                builder.build().unwrap(),
8877                RuntimeConfig {
8878                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8879                    default_model: "gpt-5.5".to_string(),
8880                    tool_allowlist: vec!["edit".to_string()],
8881                    model_profiles: std::collections::HashMap::from([(
8882                        "gpt-5.5".to_string(),
8883                        test_model_profile("gpt-5.5"),
8884                    )]),
8885                    ..RuntimeConfig::default()
8886                },
8887            )
8888            .unwrap(),
8889        )
8890    }
8891
8892    #[tokio::test]
8893    async fn runtime_and_thread_allowlists_intersect() {
8894        let requests = Arc::new(StdMutex::new(Vec::new()));
8895        let thread_root = std::env::temp_dir().join(format!(
8896            "roder-allowlist-intersect-{}",
8897            uuid::Uuid::new_v4()
8898        ));
8899        let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8900
8901        let request = captured_thread_override_request(
8902            &runtime,
8903            &requests,
8904            vec!["edit".to_string(), "write_file".to_string()],
8905            None,
8906            Vec::new(),
8907        )
8908        .await;
8909
8910        // The thread allowlist must not re-enable tools the runtime allowlist bans.
8911        let names = request
8912            .tools
8913            .iter()
8914            .map(|tool| tool.name.as_str())
8915            .collect::<Vec<_>>();
8916        assert_eq!(names, vec!["edit"]);
8917
8918        let _ = std::fs::remove_dir_all(thread_root);
8919    }
8920
8921    #[tokio::test]
8922    async fn route_tool_call_denies_tools_outside_allowlists() {
8923        let requests = Arc::new(StdMutex::new(Vec::new()));
8924        let thread_root =
8925            std::env::temp_dir().join(format!("roder-allowlist-dispatch-{}", uuid::Uuid::new_v4()));
8926        let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8927        let thread_id = runtime
8928            .create_thread_with(CreateThreadRequest {
8929                title: Some("Dispatch allowlist".to_string()),
8930                workspace: test_workspace(),
8931                workspace_id: None,
8932                root_id: None,
8933                provider: None,
8934                model: None,
8935                selection_mode: None,
8936                tool_allowlist: vec!["edit".to_string(), "write_file".to_string()],
8937                developer_instructions: None,
8938                external_tools: Vec::new(),
8939                runner: None,
8940            })
8941            .await
8942            .unwrap()
8943            .thread_id;
8944
8945        // write_file is registered and on the thread allowlist but banned by the runtime allowlist.
8946        let result = runtime
8947            .route_tool_call(
8948                &thread_id,
8949                &"turn-allowlist-dispatch".to_string(),
8950                roder_api::inference::ToolCallCompleted {
8951                    id: "call-1".to_string(),
8952                    name: "write_file".to_string(),
8953                    arguments: r#"{"path":"a.txt","content":"hi"}"#.to_string(),
8954                },
8955                None,
8956                None,
8957            )
8958            .await
8959            .unwrap();
8960
8961        assert!(result.is_error);
8962        assert!(
8963            result
8964                .result
8965                .contains("not permitted by the tool allowlist"),
8966            "unexpected result: {}",
8967            result.result
8968        );
8969
8970        let _ = std::fs::remove_dir_all(thread_root);
8971    }
8972
8973    #[tokio::test]
8974    async fn route_tool_calls_denies_agent_swarm_mixed_with_other_tools() {
8975        let requests = Arc::new(StdMutex::new(Vec::new()));
8976        let thread_root =
8977            std::env::temp_dir().join(format!("roder-swarm-exclusive-{}", uuid::Uuid::new_v4()));
8978        let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8979
8980        // A response that mixes agent_swarm with another tool is denied wholesale:
8981        // both calls get an error result with retry guidance, and no tool runs.
8982        let results = runtime
8983            .route_tool_calls(
8984                &"thread-swarm".to_string(),
8985                &"turn-swarm".to_string(),
8986                vec![
8987                    roder_api::inference::ToolCallCompleted {
8988                        id: "swarm-1".to_string(),
8989                        name: "agent_swarm".to_string(),
8990                        arguments: "{}".to_string(),
8991                    },
8992                    roder_api::inference::ToolCallCompleted {
8993                        id: "read-1".to_string(),
8994                        name: "read_file".to_string(),
8995                        arguments: "{}".to_string(),
8996                    },
8997                ],
8998                true,
8999                None,
9000                None,
9001            )
9002            .await
9003            .unwrap();
9004
9005        assert_eq!(results.len(), 2, "every tool_call_id must get a response");
9006        assert_eq!(results[0].id, "swarm-1");
9007        assert_eq!(results[1].id, "read-1");
9008        for result in &results {
9009            assert!(result.is_error);
9010            assert!(
9011                result.result.contains("only tool call"),
9012                "unexpected result: {}",
9013                result.result
9014            );
9015        }
9016
9017        let _ = std::fs::remove_dir_all(thread_root);
9018    }
9019
9020    #[tokio::test]
9021    async fn route_tool_calls_emits_agent_swarm_started_event() {
9022        let requests = Arc::new(StdMutex::new(Vec::new()));
9023        let thread_root =
9024            std::env::temp_dir().join(format!("roder-swarm-started-{}", uuid::Uuid::new_v4()));
9025        let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
9026        let mut events = runtime.subscribe_events();
9027
9028        // A single agent_swarm call (the valid shape) emits AgentSwarmStarted on
9029        // the event bus before dispatch, with the child count parsed from args.
9030        let _ = runtime
9031            .route_tool_calls(
9032                &"thread-swarm".to_string(),
9033                &"turn-swarm".to_string(),
9034                vec![roder_api::inference::ToolCallCompleted {
9035                    id: "swarm-1".to_string(),
9036                    name: "agent_swarm".to_string(),
9037                    arguments: r#"{"description":"x","prompt_template":"Read {{item}}","items":["a.rs","b.rs"]}"#
9038                        .to_string(),
9039                }],
9040                true,
9041                None,
9042                None,
9043            )
9044            .await
9045            .unwrap();
9046
9047        let mut started_child_count = None;
9048        for _ in 0..16 {
9049            let envelope = tokio::time::timeout(std::time::Duration::from_secs(2), events.recv())
9050                .await
9051                .unwrap()
9052                .unwrap();
9053            if let RoderEvent::AgentSwarmStarted(event) = envelope.event {
9054                started_child_count = Some(event.child_count);
9055                assert_eq!(event.tool_id, "swarm-1");
9056                break;
9057            }
9058        }
9059        assert_eq!(started_child_count, Some(2));
9060
9061        let _ = std::fs::remove_dir_all(thread_root);
9062    }
9063
9064    #[test]
9065    fn agent_swarm_child_count_sums_items_and_resumes() {
9066        assert_eq!(
9067            agent_swarm_child_count(r#"{"description":"x","items":["a","b","c"]}"#),
9068            3
9069        );
9070        assert_eq!(
9071            agent_swarm_child_count(
9072                r#"{"description":"x","items":["a"],"resume_agent_ids":{"id1":"continue"}}"#
9073            ),
9074            2
9075        );
9076        // Malformed input is lenient.
9077        assert_eq!(agent_swarm_child_count("not json"), 0);
9078    }
9079
9080    #[test]
9081    fn parse_swarm_counts_reads_summary_with_omitted_buckets() {
9082        let text = "<agent_swarm_result>\n<summary>completed: 2, failed: 1</summary>\n</agent_swarm_result>";
9083        assert_eq!(parse_swarm_counts(text), Some((2, 1, 0)));
9084        let text = "<agent_swarm_result>\n<summary>completed: 0</summary>\n</agent_swarm_result>";
9085        assert_eq!(parse_swarm_counts(text), Some((0, 0, 0)));
9086        // Not a swarm result.
9087        assert_eq!(parse_swarm_counts("just text"), None);
9088    }
9089
9090    /// Signals when inference starts, then waits for `proceed` and fails the stream.
9091    struct SignalledFailureEngine {
9092        started: tokio::sync::mpsc::UnboundedSender<()>,
9093        proceed: Arc<tokio::sync::Notify>,
9094    }
9095
9096    #[async_trait::async_trait]
9097    impl InferenceEngine for SignalledFailureEngine {
9098        fn id(&self) -> String {
9099            roder_api::catalog::PROVIDER_MOCK.to_string()
9100        }
9101
9102        fn capabilities(&self) -> InferenceCapabilities {
9103            InferenceCapabilities::coding_agent_default()
9104        }
9105
9106        async fn list_models(
9107            &self,
9108            _ctx: InferenceProviderContext<'_>,
9109        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
9110            Ok(roder_api::catalog::models_for_provider(
9111                roder_api::catalog::PROVIDER_MOCK,
9112                true,
9113            ))
9114        }
9115
9116        async fn stream_turn(
9117            &self,
9118            _ctx: InferenceTurnContext<'_>,
9119            _request: AgentInferenceRequest,
9120        ) -> anyhow::Result<InferenceEventStream> {
9121            let _ = self.started.send(());
9122            self.proceed.notified().await;
9123            anyhow::bail!("engine failed mid-turn")
9124        }
9125    }
9126
9127    #[tokio::test]
9128    async fn failed_turn_sweeps_pending_external_tool_calls() {
9129        let (started_tx, mut started_rx) = tokio::sync::mpsc::unbounded_channel();
9130        let proceed = Arc::new(tokio::sync::Notify::new());
9131        let mut builder = ExtensionRegistryBuilder::new();
9132        builder.inference_engine(Arc::new(SignalledFailureEngine {
9133            started: started_tx,
9134            proceed: proceed.clone(),
9135        }));
9136        let runtime =
9137            Arc::new(Runtime::new(builder.build().unwrap(), RuntimeConfig::default()).unwrap());
9138        let mut rx = runtime.subscribe_events();
9139        let turn_id = runtime
9140            .start_turn(StartTurnRequest {
9141                thread_id: "thread-sweep".to_string(),
9142                message: "go".to_string(),
9143                images: Vec::new(),
9144                provider_override: None,
9145                model_override: None,
9146                reasoning_override: None,
9147                workspace: test_workspace(),
9148                instructions: InstructionBundle {
9149                    system: None,
9150                    developer: None,
9151                    developer_context: None,
9152                },
9153                developer_context: None,
9154                task_ledger_required: false,
9155                service_tier_override: None,
9156            })
9157            .await
9158            .unwrap();
9159        tokio::time::timeout(std::time::Duration::from_secs(5), started_rx.recv())
9160            .await
9161            .unwrap()
9162            .unwrap();
9163
9164        let (tx, _pending_rx) = oneshot::channel();
9165        runtime.pending_external_tool_calls.lock().await.insert(
9166            "exttool-sweep-test".to_string(),
9167            PendingExternalToolCall {
9168                thread_id: "thread-sweep".to_string(),
9169                turn_id: turn_id.clone(),
9170                tool_id: "call-1".to_string(),
9171                tool_name: "acme_lookup".to_string(),
9172                tx,
9173            },
9174        );
9175        proceed.notify_one();
9176
9177        let outcome = tokio::time::timeout(std::time::Duration::from_secs(5), async {
9178            loop {
9179                let envelope = rx.recv().await.unwrap();
9180                if let RoderEvent::ExternalToolCallResolved(event) = envelope.event
9181                    && event.request_id == "exttool-sweep-test"
9182                {
9183                    break event.outcome;
9184                }
9185            }
9186        })
9187        .await
9188        .expect("turn failure must resolve pending external tool calls");
9189        assert_eq!(outcome, ExternalToolCallOutcome::Cancelled);
9190        assert!(runtime.pending_external_tool_calls.lock().await.is_empty());
9191    }
9192
9193    #[tokio::test]
9194    async fn thread_developer_instructions_layer_under_harness_prompt() {
9195        let requests = Arc::new(StdMutex::new(Vec::new()));
9196        let thread_root = std::env::temp_dir().join(format!(
9197            "roder-thread-instructions-{}",
9198            uuid::Uuid::new_v4()
9199        ));
9200        let mut builder = ExtensionRegistryBuilder::new();
9201        builder.inference_engine(Arc::new(SwitchCaptureEngine {
9202            requests: requests.clone(),
9203        }));
9204        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
9205            base_path: thread_root.clone(),
9206        }));
9207        builder.tool_contributor(Arc::new(ProfileToolContributor));
9208        let runtime =
9209            Arc::new(Runtime::new(builder.build().unwrap(), RuntimeConfig::default()).unwrap());
9210
9211        let request = captured_thread_override_request(
9212            &runtime,
9213            &requests,
9214            Vec::new(),
9215            Some("You are embedded in a host app.".to_string()),
9216            Vec::new(),
9217        )
9218        .await;
9219
9220        let system = request.instructions.system.expect("system instructions");
9221        assert!(system.starts_with("You are Roder"));
9222        let developer = request
9223            .instructions
9224            .developer
9225            .expect("developer instructions");
9226        assert!(developer.starts_with("You are embedded in a host app."));
9227
9228        let plain =
9229            captured_thread_override_request(&runtime, &requests, Vec::new(), None, Vec::new())
9230                .await;
9231        assert_eq!(plain.instructions.developer, None);
9232
9233        let _ = std::fs::remove_dir_all(thread_root);
9234    }
9235
9236    #[tokio::test]
9237    async fn thread_external_tools_are_advertised_and_shadow_builtins() {
9238        let requests = Arc::new(StdMutex::new(Vec::new()));
9239        let thread_root = std::env::temp_dir().join(format!(
9240            "roder-thread-external-tools-{}",
9241            uuid::Uuid::new_v4()
9242        ));
9243        let mut builder = ExtensionRegistryBuilder::new();
9244        builder.inference_engine(Arc::new(SwitchCaptureEngine {
9245            requests: requests.clone(),
9246        }));
9247        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
9248            base_path: thread_root.clone(),
9249        }));
9250        builder.tool_contributor(Arc::new(ProfileToolContributor));
9251        let runtime =
9252            Arc::new(Runtime::new(builder.build().unwrap(), RuntimeConfig::default()).unwrap());
9253
9254        let external_tools = vec![
9255            ToolSpec {
9256                name: "acme_lookup".to_string(),
9257                description: "Look up Acme workspace state.".to_string(),
9258                parameters: serde_json::json!({
9259                    "type": "object",
9260                    "properties": { "query": { "type": "string" } },
9261                    "required": ["query"]
9262                }),
9263            },
9264            ToolSpec {
9265                name: "edit".to_string(),
9266                description: "Host-managed edit.".to_string(),
9267                parameters: serde_json::json!({ "type": "object" }),
9268            },
9269        ];
9270        let request =
9271            captured_thread_override_request(&runtime, &requests, Vec::new(), None, external_tools)
9272                .await;
9273
9274        let acme = request
9275            .tools
9276            .iter()
9277            .find(|tool| tool.name == "acme_lookup")
9278            .expect("external tool advertised");
9279        assert_eq!(acme.description, "Look up Acme workspace state.");
9280        assert_eq!(acme.parameters["required"][0], "query");
9281        let edits = request
9282            .tools
9283            .iter()
9284            .filter(|tool| tool.name == "edit")
9285            .collect::<Vec<_>>();
9286        assert_eq!(edits.len(), 1, "external edit shadows the builtin");
9287        assert_eq!(edits[0].description, "Host-managed edit.");
9288
9289        let plain =
9290            captured_thread_override_request(&runtime, &requests, Vec::new(), None, Vec::new())
9291                .await;
9292        assert!(plain.tools.iter().all(|tool| tool.name != "acme_lookup"));
9293        let plain_edit = plain
9294            .tools
9295            .iter()
9296            .find(|tool| tool.name == "edit")
9297            .expect("builtin edit advertised on plain thread");
9298        assert_eq!(plain_edit.description, "edit test tool");
9299
9300        let _ = std::fs::remove_dir_all(thread_root);
9301    }
9302
9303    #[tokio::test]
9304    async fn model_switch_injects_summary_and_records_profile_segments() {
9305        let requests = Arc::new(StdMutex::new(Vec::new()));
9306        let thread_root = std::env::temp_dir().join(format!(
9307            "roder-model-switch-thread-{}",
9308            uuid::Uuid::new_v4()
9309        ));
9310        let mut builder = ExtensionRegistryBuilder::new();
9311        builder.inference_engine(Arc::new(SwitchCaptureEngine {
9312            requests: requests.clone(),
9313        }));
9314        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
9315            base_path: thread_root.clone(),
9316        }));
9317        builder.tool_contributor(Arc::new(ProfileToolContributor));
9318        let mut claude_profile = test_model_profile("claude-haiku-4-5-20251001");
9319        claude_profile.provider_family = ProviderFamily::Anthropic;
9320        claude_profile.edit_tool = Some(EDIT_TOOL_EDIT.to_string());
9321        let runtime = Arc::new(
9322            Runtime::new(
9323                builder.build().unwrap(),
9324                RuntimeConfig {
9325                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9326                    default_model: "gpt-5.5".to_string(),
9327                    model_profiles: std::collections::HashMap::from([
9328                        ("gpt-5.5".to_string(), test_model_profile("gpt-5.5")),
9329                        ("claude-haiku-4-5-20251001".to_string(), claude_profile),
9330                    ]),
9331                    ..RuntimeConfig::default()
9332                },
9333            )
9334            .unwrap(),
9335        );
9336        let thread_id = runtime
9337            .create_thread_with(CreateThreadRequest {
9338                title: Some("Model switch".to_string()),
9339                workspace: test_workspace(),
9340                workspace_id: None,
9341                root_id: None,
9342                provider: None,
9343                model: None,
9344                selection_mode: None,
9345                tool_allowlist: Vec::new(),
9346                developer_instructions: None,
9347                external_tools: Vec::new(),
9348                runner: None,
9349            })
9350            .await
9351            .unwrap()
9352            .thread_id;
9353        let mut rx = runtime.subscribe_events();
9354        for (message, model_override) in [
9355            ("first turn", None),
9356            ("second turn", Some("claude-haiku-4-5-20251001".to_string())),
9357        ] {
9358            let turn_id = runtime
9359                .start_turn(StartTurnRequest {
9360                    thread_id: thread_id.clone(),
9361                    message: message.to_string(),
9362                    images: Vec::new(),
9363                    provider_override: None,
9364                    model_override,
9365                    reasoning_override: None,
9366                    workspace: test_workspace(),
9367                    instructions: InstructionBundle::default(),
9368                    developer_context: None,
9369                    task_ledger_required: false,
9370                    service_tier_override: None,
9371                })
9372                .await
9373                .unwrap();
9374            tokio::time::timeout(std::time::Duration::from_secs(5), async {
9375                loop {
9376                    let envelope = rx.recv().await.unwrap();
9377                    if envelope.turn_id.as_deref() != Some(&turn_id) {
9378                        continue;
9379                    }
9380                    match envelope.event {
9381                        RoderEvent::TurnCompleted(_) => break,
9382                        RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9383                        _ => {}
9384                    }
9385                }
9386            })
9387            .await
9388            .unwrap();
9389        }
9390
9391        let captured = requests.lock().unwrap().clone();
9392        assert_eq!(captured.len(), 2);
9393        assert!(captured[1].transcript.iter().any(|item| {
9394            matches!(
9395                item,
9396                TranscriptItem::UserMessage(message)
9397                    if message.text.starts_with(MODEL_SWITCH_SUMMARY_PREFIX)
9398                        && message.text.contains("previous profile mock/gpt-5.5")
9399                        && message.text.contains("Current profile mock/claude-haiku-4-5-20251001")
9400                        && message.text.contains("Available tools now:")
9401            )
9402        }));
9403
9404        let snapshot = runtime
9405            .thread_store
9406            .as_ref()
9407            .unwrap()
9408            .load_thread(&thread_id)
9409            .await
9410            .unwrap()
9411            .unwrap();
9412        let trace_segments = snapshot
9413            .turns
9414            .iter()
9415            .flat_map(|turn| &turn.items)
9416            .filter(|item| {
9417                matches!(
9418                    item,
9419                    TranscriptItem::ProviderMetadata(value)
9420                        if value.get("kind").and_then(serde_json::Value::as_str)
9421                            == Some(MODEL_PROFILE_TRACE_KIND)
9422                            && value.get("segment").and_then(serde_json::Value::as_str)
9423                                == Some("assistant")
9424                )
9425            })
9426            .count();
9427        assert!(trace_segments >= 2);
9428        let _ = std::fs::remove_dir_all(thread_root);
9429    }
9430
9431    struct CountingTaskTool {
9432        calls: Arc<StdMutex<u32>>,
9433    }
9434
9435    #[async_trait::async_trait]
9436    impl ToolExecutor for CountingTaskTool {
9437        fn spec(&self) -> ToolSpec {
9438            ToolSpec {
9439                name: "task".to_string(),
9440                description: "Dispatch a test subagent.".to_string(),
9441                parameters: serde_json::json!({
9442                    "type": "object",
9443                    "properties": {
9444                        "description": { "type": "string" },
9445                        "prompt": { "type": "string" },
9446                        "parent_deadline_seconds": { "type": "integer" }
9447                    },
9448                    "required": ["description", "prompt"],
9449                    "additionalProperties": false
9450                }),
9451            }
9452        }
9453
9454        async fn execute(
9455            &self,
9456            _ctx: ToolExecutionContext,
9457            call: ToolCall,
9458        ) -> anyhow::Result<ToolResult> {
9459            *self.calls.lock().unwrap() += 1;
9460            Ok(ToolResult {
9461                id: call.id,
9462                name: call.name,
9463                text: "started child".to_string(),
9464                data: serde_json::json!({}),
9465                is_error: false,
9466            })
9467        }
9468    }
9469
9470    #[tokio::test]
9471    async fn runtime_profile_reaches_inference_request_and_turn_metadata() {
9472        let captured = Arc::new(StdMutex::new(None));
9473        let mut builder = ExtensionRegistryBuilder::new();
9474        builder.inference_engine(Arc::new(CapturingEngine {
9475            request: captured.clone(),
9476        }));
9477        let runtime = Arc::new(
9478            Runtime::new(
9479                builder.build().unwrap(),
9480                RuntimeConfig {
9481                    runtime_profile: RuntimeProfile::NonInteractive,
9482                    ..RuntimeConfig::default()
9483                },
9484            )
9485            .unwrap(),
9486        );
9487        let mut rx = runtime.subscribe_events();
9488        let turn_id = runtime
9489            .start_turn(StartTurnRequest {
9490                thread_id: "thread-profile".to_string(),
9491                message: "work unattended".to_string(),
9492                images: Vec::new(),
9493                provider_override: None,
9494                model_override: None,
9495                reasoning_override: None,
9496                workspace: test_workspace(),
9497                instructions: InstructionBundle {
9498                    system: None,
9499                    developer: Some("base developer".to_string()),
9500                    developer_context: None,
9501                },
9502                developer_context: None,
9503                task_ledger_required: false,
9504                service_tier_override: None,
9505            })
9506            .await
9507            .unwrap();
9508
9509        let mut observed_profile = None;
9510        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9511            loop {
9512                let envelope = rx.recv().await.unwrap();
9513                if envelope.turn_id.as_deref() != Some(&turn_id) {
9514                    continue;
9515                }
9516                match envelope.event {
9517                    RoderEvent::TurnStarted(event) => {
9518                        observed_profile = Some(event.runtime_profile);
9519                    }
9520                    RoderEvent::TurnCompleted(_) => break,
9521                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9522                    _ => {}
9523                }
9524            }
9525        })
9526        .await
9527        .unwrap();
9528
9529        assert_eq!(observed_profile, Some(RuntimeProfile::NonInteractive));
9530        let request = captured.lock().unwrap().clone().unwrap();
9531        assert_eq!(request.runtime.profile, RuntimeProfile::NonInteractive);
9532        let developer = request.instructions.developer.unwrap();
9533        assert!(developer.contains("base developer"));
9534        assert!(developer.contains("non-interactive profile"));
9535    }
9536
9537    #[tokio::test]
9538    async fn global_policy_mode_changes_do_not_create_runtime_thread_directory() {
9539        let workspace = runtime_test_workspace("global-policy-mode");
9540        let thread_root = workspace.join("threads");
9541        let mut builder = ExtensionRegistryBuilder::new();
9542        builder.inference_engine(Arc::new(FakeInferenceEngine));
9543        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
9544            base_path: thread_root.clone(),
9545        }));
9546        let runtime = Runtime::new(
9547            builder.build().unwrap(),
9548            RuntimeConfig {
9549                workspace: Some(workspace.display().to_string()),
9550                ..Default::default()
9551            },
9552        )
9553        .unwrap();
9554
9555        runtime
9556            .set_policy_mode(PolicyMode::AcceptAll, Some("test".to_string()))
9557            .await
9558            .unwrap();
9559
9560        assert!(!thread_root.join("runtime").exists());
9561        let _ = std::fs::remove_dir_all(workspace);
9562    }
9563
9564    #[tokio::test]
9565    async fn task_ledger_enforcement_injects_eval_reminder_before_work() {
9566        let captured = Arc::new(StdMutex::new(None));
9567        let mut builder = ExtensionRegistryBuilder::new();
9568        builder.inference_engine(Arc::new(CapturingEngine {
9569            request: captured.clone(),
9570        }));
9571        builder.tool_contributor(Arc::new(
9572            roder_ext_task_ledger::TaskLedgerToolContributor::default(),
9573        ));
9574        let runtime = Arc::new(
9575            Runtime::new(
9576                builder.build().unwrap(),
9577                RuntimeConfig {
9578                    runtime_profile: RuntimeProfile::Eval,
9579                    policy_mode: PolicyMode::Bypass,
9580                    agent_swarm_mode: false,
9581                    ultra_mode: false,
9582                    ..RuntimeConfig::default()
9583                },
9584            )
9585            .unwrap(),
9586        );
9587        let mut rx = runtime.subscribe_events();
9588        let turn_id = runtime
9589            .start_turn(StartTurnRequest {
9590                thread_id: "thread-ledger".to_string(),
9591                message: "decomposed work".to_string(),
9592                images: Vec::new(),
9593                provider_override: None,
9594                model_override: None,
9595                reasoning_override: None,
9596                workspace: test_workspace(),
9597                instructions: InstructionBundle::default(),
9598                developer_context: None,
9599                task_ledger_required: true,
9600                service_tier_override: None,
9601            })
9602            .await
9603            .unwrap();
9604
9605        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9606            loop {
9607                let envelope = rx.recv().await.unwrap();
9608                if envelope.turn_id.as_deref() == Some(&turn_id)
9609                    && matches!(envelope.event, RoderEvent::TurnCompleted(_))
9610                {
9611                    break;
9612                }
9613            }
9614        })
9615        .await
9616        .unwrap();
9617
9618        let request = captured.lock().unwrap().clone().unwrap();
9619        let developer = request.instructions.developer.unwrap();
9620        assert!(developer.contains("Task Ledger Required"));
9621        assert!(developer.contains("task_ledger.update"));
9622        let tool_names: Vec<_> = request
9623            .tools
9624            .iter()
9625            .map(|tool| tool.name.as_str())
9626            .collect();
9627        assert!(
9628            tool_names.contains(&TASK_LEDGER_TOOL_NAME),
9629            "tool names: {tool_names:?}"
9630        );
9631        assert_eq!(
9632            request.tool_choice,
9633            ToolChoice::Specific(TASK_LEDGER_TOOL_NAME.to_string())
9634        );
9635        assert_eq!(request.tools.len(), 1);
9636        assert_eq!(request.tools[0].name, TASK_LEDGER_TOOL_NAME);
9637    }
9638
9639    #[tokio::test]
9640    async fn eval_task_ledger_blocks_final_answer_until_open_items_are_completed() {
9641        let requests = Arc::new(StdMutex::new(Vec::new()));
9642        let mut builder = ExtensionRegistryBuilder::new();
9643        builder.inference_engine(Arc::new(TaskLedgerCompletionGateEngine {
9644            calls: StdMutex::new(0),
9645            requests: requests.clone(),
9646        }));
9647        builder.tool_contributor(Arc::new(
9648            roder_ext_task_ledger::TaskLedgerToolContributor::default(),
9649        ));
9650        let runtime = Arc::new(
9651            Runtime::new(
9652                builder.build().unwrap(),
9653                RuntimeConfig {
9654                    runtime_profile: RuntimeProfile::Eval,
9655                    policy_mode: PolicyMode::Bypass,
9656                    agent_swarm_mode: false,
9657                    ultra_mode: false,
9658                    ..RuntimeConfig::default()
9659                },
9660            )
9661            .unwrap(),
9662        );
9663        let mut rx = runtime.subscribe_events();
9664        let turn_id = runtime
9665            .start_turn(StartTurnRequest {
9666                thread_id: "thread-ledger-completion".to_string(),
9667                message: "write the answer file".to_string(),
9668                images: Vec::new(),
9669                provider_override: None,
9670                model_override: None,
9671                reasoning_override: None,
9672                workspace: test_workspace(),
9673                instructions: InstructionBundle::default(),
9674                developer_context: None,
9675                task_ledger_required: true,
9676                service_tier_override: None,
9677            })
9678            .await
9679            .unwrap();
9680
9681        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9682            loop {
9683                let envelope = rx.recv().await.unwrap();
9684                if envelope.turn_id.as_deref() != Some(&turn_id) {
9685                    continue;
9686                }
9687                match envelope.event {
9688                    RoderEvent::TurnCompleted(_) => break,
9689                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9690                    _ => {}
9691                }
9692            }
9693        })
9694        .await
9695        .unwrap();
9696
9697        let requests = requests.lock().unwrap().clone();
9698        assert_eq!(requests.len(), 4);
9699        assert!(requests[2].transcript.iter().any(|item| {
9700            matches!(
9701                item,
9702                TranscriptItem::UserMessage(message)
9703                    if message.text.contains("Task Ledger Completion Required")
9704                        && message.text.contains("Write /app/result.txt")
9705            )
9706        }));
9707        assert!(requests[3].transcript.iter().any(|item| {
9708            matches!(
9709                item,
9710                TranscriptItem::ToolResult(result)
9711                    if result.name.as_deref() == Some(TASK_LEDGER_TOOL_NAME)
9712                        && result.result.contains("Task ledger: 2/2 completed")
9713            )
9714        }));
9715    }
9716
9717    #[tokio::test]
9718    async fn eval_task_ledger_checkpoint_requests_scoreable_file_before_final_reserve() {
9719        let requests = Arc::new(StdMutex::new(Vec::new()));
9720        let mut builder = ExtensionRegistryBuilder::new();
9721        builder.inference_engine(Arc::new(TaskLedgerCompletionGateEngine {
9722            calls: StdMutex::new(0),
9723            requests: requests.clone(),
9724        }));
9725        builder.tool_contributor(Arc::new(
9726            roder_ext_task_ledger::TaskLedgerToolContributor::default(),
9727        ));
9728        let runtime = Arc::new(
9729            Runtime::new(
9730                builder.build().unwrap(),
9731                RuntimeConfig {
9732                    runtime_profile: RuntimeProfile::Eval,
9733                    policy_mode: PolicyMode::Bypass,
9734                    agent_swarm_mode: false,
9735                    ultra_mode: false,
9736                    turn_deadline_seconds: Some(120),
9737                    ..RuntimeConfig::default()
9738                },
9739            )
9740            .unwrap(),
9741        );
9742        let mut rx = runtime.subscribe_events();
9743        let turn_id = runtime
9744            .start_turn(StartTurnRequest {
9745                thread_id: "thread-ledger-checkpoint".to_string(),
9746                message: "write the answer file".to_string(),
9747                images: Vec::new(),
9748                provider_override: None,
9749                model_override: None,
9750                reasoning_override: None,
9751                workspace: test_workspace(),
9752                instructions: InstructionBundle::default(),
9753                developer_context: None,
9754                task_ledger_required: true,
9755                service_tier_override: None,
9756            })
9757            .await
9758            .unwrap();
9759
9760        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9761            loop {
9762                let envelope = rx.recv().await.unwrap();
9763                if envelope.turn_id.as_deref() != Some(&turn_id) {
9764                    continue;
9765                }
9766                match envelope.event {
9767                    RoderEvent::TurnCompleted(_) => break,
9768                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9769                    _ => {}
9770                }
9771            }
9772        })
9773        .await
9774        .unwrap();
9775
9776        let requests = requests.lock().unwrap().clone();
9777        assert!(requests.len() >= 2);
9778        assert!(requests[1].transcript.iter().any(|item| {
9779            matches!(
9780                item,
9781                TranscriptItem::UserMessage(message)
9782                    if message.text.contains("Scoreable Output Checkpoint")
9783                        && message.text.contains("ensure the required output file(s) exist")
9784                        && message.text.contains("Write /app/result.txt")
9785            )
9786        }));
9787    }
9788
9789    #[test]
9790    fn deadline_task_ledger_prompt_preserves_scoreable_work_instruction() {
9791        let prompt = task_ledger_deadline_completion_prompt(
9792            12,
9793            30,
9794            "Task Ledger Completion Required: write /app/result.txt, then call task_ledger.update",
9795        );
9796
9797        assert!(prompt.contains("12 seconds remain"));
9798        assert!(prompt.contains("create or update the required scoreable output files"));
9799        assert!(prompt.contains("write /app/result.txt"));
9800        assert!(prompt.contains(TASK_LEDGER_TOOL_NAME));
9801    }
9802
9803    #[test]
9804    fn scoreable_checkpoint_prompt_preserves_provisional_file_instruction() {
9805        let prompt = task_ledger_scoreable_checkpoint_prompt(
9806            120,
9807            "Task Ledger Completion Required: write /app/result.txt, then call task_ledger.update",
9808        );
9809
9810        assert!(prompt.contains("120 seconds remain"));
9811        assert!(prompt.contains("best evidence-backed answer"));
9812        assert!(prompt.contains("even if provisional"));
9813        assert!(prompt.contains("preserve that candidate"));
9814        assert!(prompt.contains("partial-coverage"));
9815        assert!(prompt.contains("write /app/result.txt"));
9816        assert!(prompt.contains(TASK_LEDGER_TOOL_NAME));
9817    }
9818
9819    #[test]
9820    fn open_task_ledger_moves_inference_timeout_to_scoreable_checkpoint() {
9821        let deadline = Some(OffsetDateTime::now_utc() + Duration::seconds(870));
9822        let transcript = vec![TranscriptItem::ToolResult(ToolResultRecord {
9823            id: "ledger-open".to_string(),
9824            name: Some(TASK_LEDGER_TOOL_NAME.to_string()),
9825            result: "Task ledger: 0/1 completed\n- pending: Write /app/result.txt [write]"
9826                .to_string(),
9827            display_payload: None,
9828            is_error: false,
9829        })];
9830
9831        let (_, action) = inference_timeout_deadline(
9832            deadline,
9833            RuntimeProfile::Eval,
9834            true,
9835            30,
9836            false,
9837            0,
9838            &transcript,
9839        )
9840        .unwrap();
9841
9842        assert_eq!(action, InferenceTimeoutAction::ScoreableCheckpoint);
9843    }
9844
9845    #[tokio::test]
9846    async fn verification_gate_forces_eval_code_changes_through_review() {
9847        let mut builder = ExtensionRegistryBuilder::new();
9848        builder.inference_engine(Arc::new(VerificationGateEngine {
9849            calls: StdMutex::new(0),
9850        }));
9851        builder.tool_contributor(Arc::new(WriteFileContributor));
9852        builder.tool_contributor(Arc::new(
9853            roder_ext_verification::VerificationToolContributor,
9854        ));
9855        let runtime = Arc::new(
9856            Runtime::new(
9857                builder.build().unwrap(),
9858                RuntimeConfig {
9859                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9860                    default_model: "mock".to_string(),
9861                    runtime_profile: RuntimeProfile::Eval,
9862                    policy_mode: PolicyMode::Bypass,
9863                    agent_swarm_mode: false,
9864                    ultra_mode: false,
9865                    ..RuntimeConfig::default()
9866                },
9867            )
9868            .unwrap(),
9869        );
9870        let mut rx = runtime.subscribe_events();
9871        let turn_id = runtime
9872            .start_turn(StartTurnRequest {
9873                thread_id: "thread-verification".to_string(),
9874                message: "write code".to_string(),
9875                images: Vec::new(),
9876                provider_override: None,
9877                model_override: None,
9878                reasoning_override: None,
9879                workspace: test_workspace(),
9880                instructions: InstructionBundle::default(),
9881                developer_context: None,
9882                task_ledger_required: false,
9883                service_tier_override: None,
9884            })
9885            .await
9886            .unwrap();
9887
9888        let mut saw_required = false;
9889        let mut saw_completed = false;
9890        let mut final_text = String::new();
9891        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9892            loop {
9893                let envelope = rx.recv().await.unwrap();
9894                if envelope.turn_id.as_deref() != Some(&turn_id) {
9895                    continue;
9896                }
9897                match envelope.event {
9898                    RoderEvent::VerificationRequired(event) => {
9899                        saw_required = true;
9900                        assert_eq!(event.changed_files, vec!["src/lib.rs"]);
9901                    }
9902                    RoderEvent::VerificationCompleted(event) => {
9903                        saw_completed = true;
9904                        assert!(event.passed);
9905                    }
9906                    RoderEvent::InferenceEventReceived(event) => {
9907                        if let InferenceEvent::MessageDelta(delta) = event.event {
9908                            final_text.push_str(&delta.text);
9909                        }
9910                    }
9911                    RoderEvent::TurnCompleted(_) => break,
9912                    _ => {}
9913                }
9914            }
9915        })
9916        .await
9917        .unwrap();
9918
9919        assert!(saw_required);
9920        assert!(saw_completed);
9921        assert!(final_text.contains("verified final"));
9922    }
9923
9924    #[tokio::test]
9925    async fn speed_policy_changes_reasoning_across_eval_model_calls_without_model_switch() {
9926        let requests = Arc::new(StdMutex::new(Vec::new()));
9927        let mut builder = ExtensionRegistryBuilder::new();
9928        builder.inference_engine(Arc::new(SpeedPolicyEngine {
9929            calls: StdMutex::new(0),
9930            requests: requests.clone(),
9931        }));
9932        builder.tool_contributor(Arc::new(WriteFileContributor));
9933        builder.tool_contributor(Arc::new(
9934            roder_ext_verification::VerificationToolContributor,
9935        ));
9936        let runtime = Arc::new(
9937            Runtime::new(
9938                builder.build().unwrap(),
9939                RuntimeConfig {
9940                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9941                    default_model: "gpt-5.5".to_string(),
9942                    runtime_profile: RuntimeProfile::Eval,
9943                    policy_mode: PolicyMode::Bypass,
9944                    agent_swarm_mode: false,
9945                    ultra_mode: false,
9946                    ..RuntimeConfig::default()
9947                },
9948            )
9949            .unwrap(),
9950        );
9951        let mut rx = runtime.subscribe_events();
9952        let turn_id = runtime
9953            .start_turn(StartTurnRequest {
9954                thread_id: "thread-speed-policy".to_string(),
9955                message: "write code".to_string(),
9956                images: Vec::new(),
9957                provider_override: None,
9958                model_override: None,
9959                reasoning_override: None,
9960                workspace: test_workspace(),
9961                instructions: InstructionBundle::default(),
9962                developer_context: None,
9963                task_ledger_required: false,
9964                service_tier_override: None,
9965            })
9966            .await
9967            .unwrap();
9968
9969        let mut saw_speed_policy_event = false;
9970        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9971            loop {
9972                let envelope = rx.recv().await.unwrap();
9973                if envelope.turn_id.as_deref() != Some(&turn_id) {
9974                    continue;
9975                }
9976                match envelope.event {
9977                    RoderEvent::InferenceStarted(event) => {
9978                        if event.speed_policy.is_some() {
9979                            saw_speed_policy_event = true;
9980                        }
9981                    }
9982                    RoderEvent::TurnCompleted(_) => break,
9983                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9984                    _ => {}
9985                }
9986            }
9987        })
9988        .await
9989        .unwrap();
9990
9991        let requests = requests.lock().unwrap().clone();
9992        assert!(saw_speed_policy_event);
9993        assert!(requests.len() >= 4);
9994        assert!(requests.iter().all(|request| {
9995            request.model.provider == roder_api::catalog::PROVIDER_MOCK
9996                && request.model.model == "gpt-5.5"
9997        }));
9998        assert_eq!(
9999            requests[0].runtime.speed_policy.as_ref().map(|d| d.phase),
10000            Some(roder_api::inference::SpeedPolicyPhase::Orientation)
10001        );
10002        assert_eq!(requests[0].reasoning.level.as_deref(), Some(REASONING_HIGH));
10003        assert_eq!(
10004            requests[1].runtime.speed_policy.as_ref().map(|d| d.phase),
10005            Some(roder_api::inference::SpeedPolicyPhase::Execution)
10006        );
10007        assert_eq!(requests[1].reasoning.level.as_deref(), Some(REASONING_LOW));
10008        assert_eq!(
10009            requests[2].runtime.speed_policy.as_ref().map(|d| d.phase),
10010            Some(roder_api::inference::SpeedPolicyPhase::Verification)
10011        );
10012        assert_eq!(requests[2].reasoning.level.as_deref(), Some(REASONING_HIGH));
10013        assert_eq!(
10014            requests[2]
10015                .metadata
10016                .pointer("/speedPolicy/phase")
10017                .and_then(serde_json::Value::as_str),
10018            Some("verification")
10019        );
10020    }
10021
10022    #[tokio::test]
10023    async fn deadline_turn_timeout_emits_partial_result_and_clears_active_turn() {
10024        let mut builder = ExtensionRegistryBuilder::new();
10025        builder.inference_engine(Arc::new(DeadlineEngine));
10026        let runtime = Arc::new(
10027            Runtime::new(
10028                builder.build().unwrap(),
10029                RuntimeConfig {
10030                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
10031                    default_model: "mock".to_string(),
10032                    runtime_profile: RuntimeProfile::Eval,
10033                    turn_deadline_seconds: Some(1),
10034                    ..RuntimeConfig::default()
10035                },
10036            )
10037            .unwrap(),
10038        );
10039        let mut rx = runtime.subscribe_events();
10040        let turn_id = runtime
10041            .start_turn(StartTurnRequest {
10042                thread_id: "thread-deadline".to_string(),
10043                message: "slow work".to_string(),
10044                images: Vec::new(),
10045                provider_override: None,
10046                model_override: None,
10047                reasoning_override: None,
10048                workspace: test_workspace(),
10049                instructions: InstructionBundle::default(),
10050                developer_context: None,
10051                task_ledger_required: false,
10052                service_tier_override: None,
10053            })
10054            .await
10055            .unwrap();
10056
10057        let mut saw_partial = false;
10058        let mut saw_deadline = false;
10059        let mut failed_kind = None;
10060        tokio::time::timeout(std::time::Duration::from_secs(5), async {
10061            loop {
10062                let envelope = rx.recv().await.unwrap();
10063                if envelope.turn_id.as_deref() != Some(&turn_id) {
10064                    continue;
10065                }
10066                match envelope.event {
10067                    RoderEvent::TurnPartialResult(event) => {
10068                        saw_partial = event.summary.contains("partial turn state");
10069                    }
10070                    RoderEvent::TurnDeadlineExceeded(event) => {
10071                        saw_deadline = event.partial_result.contains("transcript items");
10072                    }
10073                    RoderEvent::TurnFailed(event) => {
10074                        failed_kind = event.error_kind;
10075                        break;
10076                    }
10077                    _ => {}
10078                }
10079            }
10080        })
10081        .await
10082        .unwrap();
10083
10084        for _ in 0..20 {
10085            if !runtime.active_turns.read().await.contains_key(&turn_id) {
10086                break;
10087            }
10088            tokio::time::sleep(std::time::Duration::from_millis(10)).await;
10089        }
10090        assert!(saw_partial);
10091        assert!(saw_deadline);
10092        assert_eq!(failed_kind.as_deref(), Some("deadline_timeout"));
10093        assert!(!runtime.active_turns.read().await.contains_key(&turn_id));
10094    }
10095
10096    #[tokio::test]
10097    async fn deadline_skips_subagent_task_when_remaining_budget_is_too_low() {
10098        let calls = Arc::new(StdMutex::new(0));
10099        let mut builder = ExtensionRegistryBuilder::new();
10100        builder.inference_engine(Arc::new(CapturingEngine {
10101            request: Arc::new(StdMutex::new(None)),
10102        }));
10103        let task_tool = Arc::new(CountingTaskTool {
10104            calls: calls.clone(),
10105        });
10106        builder.tool_contributor(Arc::new(TestToolContributor { tool: task_tool }));
10107        let runtime = Arc::new(
10108            Runtime::new(
10109                builder.build().unwrap(),
10110                RuntimeConfig {
10111                    policy_mode: PolicyMode::Bypass,
10112                    agent_swarm_mode: false,
10113                    ultra_mode: false,
10114                    ..RuntimeConfig::default()
10115                },
10116            )
10117            .unwrap(),
10118        );
10119
10120        let result = runtime
10121            .route_tool_call(
10122                &"thread-deadline-task".to_string(),
10123                &"turn-deadline-task".to_string(),
10124                ToolCallCompleted {
10125                    id: "task-1".to_string(),
10126                    name: "task".to_string(),
10127                    arguments: serde_json::json!({
10128                        "description": "inspect",
10129                        "prompt": "read"
10130                    })
10131                    .to_string(),
10132                },
10133                None,
10134                Some(OffsetDateTime::now_utc() + Duration::seconds(1)),
10135            )
10136            .await
10137            .unwrap();
10138
10139        assert!(result.is_error);
10140        assert!(result.result.contains("deadline policy skipped"));
10141        assert_eq!(*calls.lock().unwrap(), 0);
10142    }
10143
10144    struct TestToolContributor {
10145        tool: Arc<dyn ToolExecutor>,
10146    }
10147
10148    impl ToolContributor for TestToolContributor {
10149        fn id(&self) -> String {
10150            "test-tool".to_string()
10151        }
10152
10153        fn contribute(&self, registry: &mut ToolRegistry) -> anyhow::Result<()> {
10154            registry.register(self.tool.clone())
10155        }
10156    }
10157
10158    #[tokio::test]
10159    async fn agent_swarm_mode_override_is_per_thread() {
10160        let runtime = Runtime::fake().unwrap();
10161        let trigger = roder_api::subagents::AgentSwarmModeTrigger::Manual;
10162
10163        // Off everywhere by default.
10164        assert!(
10165            !runtime
10166                .effective_agent_swarm_mode_for_thread("thread-a")
10167                .await
10168        );
10169        assert!(
10170            !runtime
10171                .effective_agent_swarm_mode_for_thread("thread-b")
10172                .await
10173        );
10174
10175        // Enabling on thread-a does not leak into thread-b.
10176        assert!(
10177            runtime
10178                .set_agent_swarm_mode_for_thread("thread-a", true, trigger)
10179                .await
10180        );
10181        assert!(
10182            runtime
10183                .effective_agent_swarm_mode_for_thread("thread-a")
10184                .await
10185        );
10186        assert!(
10187            !runtime
10188                .effective_agent_swarm_mode_for_thread("thread-b")
10189                .await
10190        );
10191
10192        // A per-thread `off` override wins over a runtime-global `on` default.
10193        runtime.set_agent_swarm_mode(true, trigger).await.unwrap();
10194        runtime
10195            .set_agent_swarm_mode_for_thread("thread-b", false, trigger)
10196            .await;
10197        assert!(
10198            !runtime
10199                .effective_agent_swarm_mode_for_thread("thread-b")
10200                .await,
10201            "explicit per-thread off overrides the global on default"
10202        );
10203        // A thread with no override still follows the global default.
10204        assert!(
10205            runtime
10206                .effective_agent_swarm_mode_for_thread("thread-c")
10207                .await,
10208            "threads without an override follow the runtime-global default"
10209        );
10210    }
10211
10212    #[tokio::test]
10213    async fn set_agent_swarm_mode_for_thread_emits_event_with_real_thread_id() {
10214        let runtime = Runtime::fake().unwrap();
10215        let mut events = runtime.subscribe_events();
10216        runtime
10217            .set_agent_swarm_mode_for_thread(
10218                "thread-xyz",
10219                true,
10220                roder_api::subagents::AgentSwarmModeTrigger::Task,
10221            )
10222            .await;
10223        let mut saw = false;
10224        for _ in 0..8 {
10225            let envelope = tokio::time::timeout(std::time::Duration::from_secs(2), events.recv())
10226                .await
10227                .unwrap()
10228                .unwrap();
10229            if let RoderEvent::AgentSwarmModeChanged(event) = envelope.event {
10230                assert_eq!(event.thread_id, "thread-xyz");
10231                assert!(event.enabled);
10232                assert_eq!(
10233                    event.trigger,
10234                    roder_api::subagents::AgentSwarmModeTrigger::Task
10235                );
10236                saw = true;
10237                break;
10238            }
10239        }
10240        assert!(
10241            saw,
10242            "expected an AgentSwarmModeChanged event for the thread"
10243        );
10244    }
10245
10246    #[tokio::test]
10247    async fn ultra_mode_override_is_per_thread() {
10248        let runtime = Runtime::fake().unwrap();
10249        let trigger = roder_api::subagents::UltraModeTrigger::Manual;
10250
10251        assert!(!runtime.effective_ultra_mode_for_thread("thread-a").await);
10252        assert!(!runtime.effective_ultra_mode_for_thread("thread-b").await);
10253
10254        assert!(
10255            runtime
10256                .set_ultra_mode_for_thread("thread-a", true, trigger)
10257                .await
10258        );
10259        assert!(runtime.effective_ultra_mode_for_thread("thread-a").await);
10260        assert!(!runtime.effective_ultra_mode_for_thread("thread-b").await);
10261
10262        runtime.set_ultra_mode(true, trigger).await.unwrap();
10263        runtime
10264            .set_ultra_mode_for_thread("thread-b", false, trigger)
10265            .await;
10266        assert!(
10267            !runtime.effective_ultra_mode_for_thread("thread-b").await,
10268            "explicit per-thread off overrides the global on default"
10269        );
10270        assert!(
10271            runtime.effective_ultra_mode_for_thread("thread-c").await,
10272            "threads without an override follow the runtime-global default"
10273        );
10274    }
10275
10276    #[tokio::test]
10277    async fn set_ultra_mode_for_thread_emits_event_with_real_thread_id() {
10278        let runtime = Runtime::fake().unwrap();
10279        let mut events = runtime.subscribe_events();
10280        runtime
10281            .set_ultra_mode_for_thread(
10282                "thread-ultra",
10283                true,
10284                roder_api::subagents::UltraModeTrigger::Task,
10285            )
10286            .await;
10287        let mut saw = false;
10288        for _ in 0..8 {
10289            let envelope = tokio::time::timeout(std::time::Duration::from_secs(2), events.recv())
10290                .await
10291                .unwrap()
10292                .unwrap();
10293            if let RoderEvent::UltraModeChanged(event) = envelope.event {
10294                assert_eq!(event.thread_id, "thread-ultra");
10295                assert!(event.enabled);
10296                assert_eq!(event.trigger, roder_api::subagents::UltraModeTrigger::Task);
10297                saw = true;
10298                break;
10299            }
10300        }
10301        assert!(saw, "expected an UltraModeChanged event for the thread");
10302    }
10303}
10304
10305#[cfg(test)]
10306#[path = "runtime/mailbox_test_helpers.rs"]
10307mod mailbox_test_helpers;