Skip to main content

roder_core/
runtime.rs

1#[path = "runtime/sampling.rs"]
2pub(crate) mod sampling;
3#[path = "tool_advertisement.rs"]
4mod tool_advertisement;
5use sampling::{
6    SamplingPreempted, SamplingRetry, completed_call_replayed, preemptible, wait_for_steer,
7};
8#[path = "runtime/sampling_output.rs"]
9mod sampling_output;
10use sampling_output::{OutputContext, SamplingOutput};
11mod compaction_template;
12#[path = "runtime/eager_tools.rs"]
13mod eager_tools;
14mod execution_receipts;
15mod owner_handoff;
16mod thread_admission;
17#[path = "runtime/tool_approvals.rs"]
18mod tool_approvals;
19#[path = "runtime/turn_start.rs"]
20mod turn_start;
21use eager_tools::EagerTools;
22#[path = "runtime/patch_progress.rs"]
23mod patch_progress;
24use patch_progress::PatchProgressTracker;
25
26use std::collections::{HashMap, HashSet};
27use std::path::PathBuf;
28use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
29use std::sync::{Arc, Weak};
30
31use anyhow::Context;
32use futures::StreamExt;
33use futures::future::{AbortHandle, Abortable, BoxFuture, try_join_all};
34use roder_api::catalog::{
35    EDIT_TOOL_EDIT, EDIT_TOOL_PATCH, PROVIDER_GEMINI, REASONING_NONE, REASONING_ULTRA,
36    built_in_model_profile, built_in_model_profile_for_provider, lookup_model,
37    model_supports_reasoning_effort,
38};
39use roder_api::context::PolicyGate;
40use roder_api::events::*;
41use roder_api::extension::ExtensionRegistry;
42use roder_api::inference::{
43    AgentInferenceRequest, HostedWebSearchConfig, HostedWebSearchMode, InferenceEngine,
44    InferenceEvent, InferenceTurnContext, InstructionBundle, ModelHarnessProfile,
45    ModelSchemaPolicy, ModelSelection, OutputConfig, ProviderTurnCleanup, ReasoningConfig,
46    RuntimeHints, RuntimeProfile, TokenUsage, ToolCallCompleted, ToolSearchConfig,
47    ToolSearchConfigOverlay, finish_reason_from_stop_reason,
48};
49use roder_api::inference_routing::{InferenceRoutingOutcome, ModelSelectionMode};
50use roder_api::lifecycle::{
51    LifecycleMetricsSnapshot, TurnCleanupOwnership, TurnCleanupState, TurnLifecycleReason,
52    TurnLifecycleRecord, TurnLifecycleSnapshot, TurnLifecycleState, turn_lifecycle_snapshot,
53};
54use roder_api::policy_mode::{PolicyDecision, PolicyMode};
55use roder_api::reliability::{
56    ReliabilityContext, ReliabilityDetails, ReliabilityErrorClass, ReliabilityLimitDecision,
57    ReliabilityLimitRecorded, ReliabilityRequestPolicy, ReliabilityRetryDecision,
58    ReliabilityRetryRecorded, provider_retry_delay_ms,
59};
60use roder_api::remote_runner::{
61    RemoteRunnerProvider, RemoteRunnerSession, RemoteWorkspace, RunnerDestination,
62    RunnerSessionState, ThreadRunnerBinding,
63};
64use roder_api::subagents::SubagentDefinition;
65use roder_api::teams::{
66    TeamId, TeamMailboxMessage, TeamMailboxMessageKind, TeamMemberDescriptor, TeamMemberRole,
67    TeamMemberStatus,
68};
69use roder_api::thread::{
70    ThreadItemEvent, ThreadItemEventKind, ThreadMetadata, ThreadSnapshot, ThreadStore,
71    ThreadUsageMetadata, is_synthetic_event_thread_id, validate_thread_workspace,
72};
73use roder_api::tools::{ToolCall, ToolChoice, ToolExecutionContext, ToolRegistry, ToolResult};
74use roder_api::transcript::{
75    AssistantMessage, ErrorRecord, InputImage, ReasoningSummary, ToolCallRecord, ToolResultRecord,
76    TranscriptItem, UserMessage,
77};
78use roder_sandbox::ScopedFilesystem;
79use roder_sandbox::process::LocalProcessRunner;
80use roder_skills::{SkillRegistry, SkillRegistryOptions};
81use time::{Duration, OffsetDateTime};
82use tokio::sync::{Mutex, Notify, RwLock, oneshot};
83
84mod codex_v2;
85
86use crate::artifacts::{
87    ContextArtifactStore as FilesystemContextArtifactStore, default_context_artifact_dir,
88};
89use crate::browser_routing::apply_browser_routing;
90use crate::bus::EventBus;
91use crate::dynamic_workflows::{
92    DynamicWorkflowEffortProfile, RuntimeDynamicWorkflowConfig, WorkflowTriggerDecision,
93    classify_workflow_trigger, ultracode_reasoning_level_for_model,
94};
95use crate::fake_provider::FakeInferenceEngine;
96use crate::goals::RuntimeGoalController;
97use crate::inference_routing::{
98    InferenceRoutingRequest, RuntimeInferenceRouterConfig, collect_inference_routing_candidates,
99    route_inference_selection, transcript_failure_count_since,
100};
101use crate::instructions::{
102    apply_agent_swarm_mode, apply_codex_multi_agent_mode, apply_model_instruction_overlay,
103    apply_parallel_web_tools, apply_plan_mode, apply_runtime_profile, apply_task_ledger_required,
104    apply_thread_developer_instructions, apply_turn_developer_context,
105};
106use crate::policy_gate::DefaultPolicyGate;
107use crate::reliability::{
108    ReliabilityLimitHit, RuntimeReliabilityConfig, TurnReliabilityState,
109    provider_stream_retry_cause,
110};
111pub use crate::speed_policy::RuntimeSpeedPolicyConfig;
112use crate::speed_policy::{SpeedPolicyState, reasoning_from_decision};
113use crate::subagent_traces::{RuntimeAgentSwarmProgressSink, RuntimeSubagentTraceSink};
114use crate::teams::{TeamManager, TeamMemberStartRequest, TeamStartRequest, TeamState};
115use crate::thread_item_cache::{ThreadItemCache, ThreadItemCacheEntry};
116use crate::tool_result_images::ToolImages;
117use crate::verification_gate::VerificationGateState;
118
119const MAX_TOOL_ROUNDS_PER_TURN: usize = 1024;
120/// Injected when `continue_on_failure_limit` turns a consecutive-tool-failure
121/// limit into a continuation, or as the empty-tool-call persistence nudge, so
122/// the model keeps working instead of ending the turn early.
123const RELIABILITY_CONTINUATION_PROMPT: &str = "Verify the task is fully complete. If any part remains unfinished or unverified, keep working using the available tools — try an alternative approach if one is failing. Only stop once the solution is complete and verified; if it is already complete, restate your final answer.";
124/// Capacity of the runtime event broadcast ring buffer. The TUI drains this on
125/// its render loop, which during an active turn only wakes at the ~6 FPS status
126/// animation cadence (backend events do not wake the input poll), so up to
127/// ~166ms of events buffer between drains. A reasoning-high provider that runs
128/// its whole tool loop inside one turn (e.g. claude-code) streams a firehose of
129/// `InferenceEventReceived` deltas plus per-step tool/thinking events; the old
130/// 1024-slot buffer overflowed in those windows and silently dropped tool and
131/// thinking rows from the live view. Sized for generous headroom across bursts
132/// and brief render stalls.
133const EVENT_BUS_CAPACITY: usize = 16_384;
134
135/// Turn id reported to session-scoped local hooks. `SessionStart` and
136/// `SessionEnd` bracket the session rather than any one turn, but the hook
137/// payload carries a turn id, so they share this synthetic value.
138const SESSION_HOOK_TURN_ID: &str = "session";
139pub(crate) const FINAL_ANSWER_PHASE: &str = "final_answer";
140pub(crate) const TASK_LEDGER_TOOL_NAME: &str = "task_ledger.update";
141const TASK_LEDGER_COMPLETION_REMINDER_LIMIT: u8 = 2;
142const TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS: u64 = 180;
143const TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT: u8 = 1;
144pub(crate) const MIN_CHILD_DEADLINE_SECONDS: u64 = 2;
145const MODEL_PROFILE_TRACE_KIND: &str = "model_profile_segment";
146const MODEL_SWITCH_SUMMARY_PREFIX: &str = "Model switch summary:";
147const PROVIDER_CLEANUP_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5);
148
149#[derive(Clone, Copy, Debug, Eq, PartialEq)]
150enum InferenceTimeoutAction {
151    ScoreableCheckpoint,
152    Finalization,
153}
154
155#[derive(Debug, Clone)]
156pub struct RuntimeConfig {
157    pub default_provider: String,
158    pub default_model: String,
159    pub reasoning: Option<String>,
160    pub auto_compact_token_limit: Option<u32>,
161    pub file_backed_dynamic_context: bool,
162    pub hosted_web_search: HostedWebSearchConfig,
163    pub tool_search: ToolSearchConfig,
164    pub provider_tool_search: HashMap<String, ToolSearchConfigOverlay>,
165    pub model_tool_search: HashMap<String, ToolSearchConfigOverlay>,
166    pub model_edit_tools: HashMap<String, String>,
167    pub model_parallel_tool_calls: HashMap<String, bool>,
168    pub model_profiles: HashMap<String, ModelHarnessProfile>,
169    pub tool_allowlist: Vec<String>,
170    /// Seconds a host-executed external tool call may stay unresolved before it fails with a timeout error.
171    pub external_tool_timeout_seconds: u64,
172    pub command_shell: String,
173    pub workspace: Option<String>,
174    pub policy_mode: PolicyMode,
175    /// Whether agent-swarm mode is active (roadmap 104). When on, the runtime
176    /// injects the swarm reminder into each turn's developer instructions so any
177    /// client benefits, not just the TUI.
178    pub agent_swarm_mode: bool,
179    /// Whether ultra mode is active. When on, the runtime injects proactive
180    /// multi-agent delegation policy for any model (not only Codex Sol/Terra
181    /// Ultra effort). Selecting Ultra reasoning on Sol/Terra still enables
182    /// proactive multi-agent for that model without requiring this flag.
183    pub ultra_mode: bool,
184    pub runtime_profile: RuntimeProfile,
185    pub inference_router: RuntimeInferenceRouterConfig,
186    pub speed_policy: RuntimeSpeedPolicyConfig,
187    pub dynamic_workflows: RuntimeDynamicWorkflowConfig,
188    pub reliability: RuntimeReliabilityConfig,
189    /// Positive wall-clock limit for a turn in any runtime profile. `None` or zero is unbounded.
190    pub turn_deadline_seconds: Option<u64>,
191    pub remote_runner_destination: Option<RunnerDestination>,
192    pub team_data_dir: Option<PathBuf>,
193    pub roadmap_data_dir: Option<PathBuf>,
194    pub media_generation: crate::media_generation::RuntimeMediaGenerationConfig,
195    /// `[review]` settings: the review sub-turn and its publishers.
196    pub review: crate::review::RuntimeReviewConfig,
197}
198
199impl Default for RuntimeConfig {
200    fn default() -> Self {
201        Self {
202            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
203            default_model: "mock".to_string(),
204            reasoning: None,
205            auto_compact_token_limit: None,
206            file_backed_dynamic_context: true,
207            hosted_web_search: HostedWebSearchConfig::cached(),
208            tool_search: ToolSearchConfig::default(),
209            provider_tool_search: HashMap::new(),
210            model_tool_search: HashMap::new(),
211            model_edit_tools: HashMap::new(),
212            model_parallel_tool_calls: HashMap::new(),
213            model_profiles: HashMap::new(),
214            tool_allowlist: Vec::new(),
215            external_tool_timeout_seconds: DEFAULT_EXTERNAL_TOOL_TIMEOUT_SECONDS,
216            command_shell: roder_api::command_shell::default_command_shell(),
217            workspace: None,
218            policy_mode: PolicyMode::Default,
219            agent_swarm_mode: false,
220            ultra_mode: false,
221            runtime_profile: RuntimeProfile::Interactive,
222            inference_router: RuntimeInferenceRouterConfig::default(),
223            speed_policy: RuntimeSpeedPolicyConfig::default(),
224            dynamic_workflows: RuntimeDynamicWorkflowConfig::default(),
225            reliability: RuntimeReliabilityConfig::default(),
226            turn_deadline_seconds: None,
227            remote_runner_destination: None,
228            team_data_dir: None,
229            roadmap_data_dir: None,
230            media_generation: crate::media_generation::RuntimeMediaGenerationConfig::default(),
231            review: crate::review::RuntimeReviewConfig::default(),
232        }
233    }
234}
235
236#[derive(Debug, Clone)]
237pub struct StartTurnRequest {
238    pub thread_id: ThreadId,
239    pub message: String,
240    pub images: Vec<InputImage>,
241    pub provider_override: Option<String>,
242    pub model_override: Option<String>,
243    pub reasoning_override: Option<String>,
244    pub workspace: String,
245    pub instructions: InstructionBundle,
246    /**
247     * Per-turn developer-authority context for this turn's InstructionBundle.
248     * Applies to every inference round of the turn, is never written to
249     * thread state. Automatic goal continuations retain it; later explicit
250     * turns receive only the context supplied by their host.
251     */
252    pub developer_context: Option<String>,
253    pub task_ledger_required: bool,
254    /**
255     * Per-turn provider service tier (OpenAI `service_tier`, e.g.
256     * `"priority"` for Fast mode). Carried to every inference round of the
257     * turn as `RuntimeHints::service_tier`; `None` keeps the provider
258     * default. Providers without a tier concept ignore it.
259     */
260    pub service_tier_override: Option<String>,
261}
262
263#[derive(Debug, Clone)]
264pub struct CreateThreadRequest {
265    pub title: Option<String>,
266    pub workspace: String,
267    pub workspace_id: Option<String>,
268    pub root_id: Option<String>,
269    pub provider: Option<String>,
270    pub model: Option<String>,
271    pub selection_mode: Option<ModelSelectionMode>,
272    /// Per-thread tool filter applied on top of the runtime allowlist. Empty = no filtering.
273    pub tool_allowlist: Vec<String>,
274    /// Host-supplied instructions added to the developer slot of every turn's inference request.
275    pub developer_instructions: Option<String>,
276    /// Host-executed tool specs advertised to the model on every turn of this thread.
277    pub external_tools: Vec<roder_api::tools::ToolSpec>,
278    /// Explicit remote-runner binding for the thread's native coding tools.
279    pub runner: Option<ThreadRunnerSelection>,
280}
281
282/**
283 * Thread-level remote-runner selection. The destination config is persisted
284 * with the thread, so secrets must reach the provider through its
285 * environment, not this config.
286 */
287#[derive(Debug, Clone)]
288pub struct ThreadRunnerSelection {
289    pub provider_id: String,
290    pub config: serde_json::Value,
291    /// Absolute path on the runner used as the thread's coding-tool workspace root.
292    pub workspace: String,
293    /**
294     * Extra absolute runner paths file reads may resolve under, beyond
295     * `workspace`. Writes and the working directory stay confined to
296     * `workspace`.
297     */
298    pub read_roots: Vec<String>,
299}
300
301#[derive(Debug, Clone, PartialEq, Eq)]
302pub struct PendingPlanExit {
303    pub thread_id: ThreadId,
304    pub turn_id: TurnId,
305    pub request_id: String,
306    pub target_mode: PolicyMode,
307    pub plan_summary: Option<String>,
308    pub next_steps: Vec<String>,
309    pub requested_at: OffsetDateTime,
310    pub expires_at: Option<OffsetDateTime>,
311}
312
313pub(crate) struct PendingToolApproval {
314    pub(crate) thread_id: ThreadId,
315    pub(crate) turn_id: TurnId,
316    pub(crate) tool_id: String,
317    pub(crate) tool_name: String,
318    pub(crate) call: roder_api::tools::ToolCall,
319    pub(crate) context: Option<ToolExecutionContext>,
320    pub(crate) tx: oneshot::Sender<bool>,
321}
322
323pub(crate) struct PendingUserInput {
324    pub(crate) thread_id: ThreadId,
325    pub(crate) turn_id: TurnId,
326    pub(crate) tx: oneshot::Sender<serde_json::Value>,
327}
328
329/// Host answer to an external tool call delivered via `tools/resolve`.
330#[derive(Debug, Clone, PartialEq, Eq)]
331pub struct ExternalToolResolution {
332    pub output: String,
333    pub is_error: bool,
334}
335
336pub(crate) struct PendingExternalToolCall {
337    pub(crate) thread_id: ThreadId,
338    pub(crate) turn_id: TurnId,
339    pub(crate) tool_id: String,
340    pub(crate) tool_name: String,
341    pub(crate) tx: oneshot::Sender<ExternalToolResolution>,
342}
343
344#[derive(Clone)]
345struct ActiveTurnHandle {
346    thread_id: ThreadId,
347    abort: AbortHandle,
348    steers: Arc<Mutex<Vec<QueuedTurnSteer>>>,
349    steer_changed: tokio::sync::watch::Sender<u64>,
350    drain: Arc<TurnDrainHandle>,
351}
352
353/// Tracks a task after it has been interrupted and removed from the interactive
354/// active-turn map. This lets users start a follow-up turn immediately while a
355/// bounded runtime shutdown can still await the original task's cleanup.
356struct TurnDrainHandle {
357    thread_id: ThreadId,
358    interrupt_requested: AtomicBool,
359    interrupt_reason: Mutex<Option<TurnLifecycleReason>>,
360    completed: AtomicBool,
361    completed_notify: Notify,
362}
363
364/// Result of a bounded runtime drain. `Clean` proves the runtime's own turn
365/// tasks have reached their cleanup paths; provider-specific child-process
366/// reaping remains an explicit provider capability.
367#[derive(Debug, Clone, PartialEq, Eq)]
368pub enum RuntimeDrainOutcome {
369    Clean {
370        interrupted_turn_ids: Vec<TurnId>,
371    },
372    DeadlineExceeded {
373        interrupted_turn_ids: Vec<TurnId>,
374        remaining_turn_ids: Vec<TurnId>,
375    },
376    /// At least one lifecycle transition initiated by this drain could not be
377    /// durably appended. The runtime still made its best cleanup attempt, but
378    /// callers must not claim a persisted terminal state as proof of recovery.
379    PersistenceFailed {
380        interrupted_turn_ids: Vec<TurnId>,
381        remaining_turn_ids: Vec<TurnId>,
382    },
383}
384
385#[derive(Clone)]
386struct QueuedTurnSteer {
387    message: UserMessage,
388    mailbox_ack: Option<MailboxDeliveryAck>,
389}
390
391#[derive(Clone)]
392struct MailboxDeliveryAck {
393    team_id: TeamId,
394    message_ids: Vec<String>,
395}
396
397#[derive(Clone)]
398struct InheritedTurnContext {
399    workspace: String,
400    instructions: InstructionBundle,
401    developer_context: Option<String>,
402}
403
404#[derive(Debug, Clone, Default, PartialEq, Eq)]
405pub struct ThreadActivity {
406    pub active_turn_id: Option<TurnId>,
407    pub active_flags: Vec<String>,
408}
409
410/// Per-thread settings persisted at thread creation and applied to every turn.
411#[derive(Debug, Clone, Default)]
412pub(crate) struct ThreadTurnOverrides {
413    pub(crate) tool_allowlist: Vec<String>,
414    pub(crate) developer_instructions: Option<String>,
415    pub(crate) external_tools: Vec<roder_api::tools::ToolSpec>,
416}
417
418#[derive(Debug, Clone, Copy, PartialEq, Eq)]
419pub(crate) enum TurnRunOutcome {
420    Completed,
421    Stopped,
422}
423
424impl PendingPlanExit {
425    pub fn new(
426        thread_id: ThreadId,
427        turn_id: TurnId,
428        request_id: String,
429        target_mode: PolicyMode,
430        plan_summary: Option<String>,
431        next_steps: Vec<String>,
432    ) -> Self {
433        let requested_at = OffsetDateTime::now_utc();
434        Self {
435            thread_id,
436            turn_id,
437            request_id,
438            target_mode,
439            plan_summary,
440            next_steps,
441            requested_at,
442            expires_at: Some(requested_at + default_plan_exit_timeout()),
443        }
444    }
445
446    pub fn is_expired(&self, now: OffsetDateTime) -> bool {
447        self.expires_at.is_some_and(|expires_at| now >= expires_at)
448    }
449}
450
451pub fn default_plan_exit_timeout() -> Duration {
452    Duration::minutes(10)
453}
454
455pub const DEFAULT_EXTERNAL_TOOL_TIMEOUT_SECONDS: u64 = 300;
456
457fn format_mailbox_messages(team: &TeamState, messages: &[TeamMailboxMessage]) -> String {
458    messages
459        .iter()
460        .map(|message| {
461            let recipient = team
462                .members
463                .iter()
464                .find(|member| member.id == message.to_member_id)
465                .map(canonical_team_member_path)
466                .unwrap_or_else(|| format!("/root/{}", message.to_member_id));
467            let sender = message
468                .from_member_id
469                .as_deref()
470                .and_then(|id| team.members.iter().find(|member| member.id == id))
471                .map(canonical_team_member_path)
472                .unwrap_or_else(|| "/root".to_string());
473            let message_type = match message.kind {
474                TeamMailboxMessageKind::Message => "MESSAGE",
475                TeamMailboxMessageKind::NewTask => "NEW_TASK",
476                TeamMailboxMessageKind::FinalAnswer => "FINAL_ANSWER",
477            };
478            format!(
479                "Message Type: {message_type}\nTask name: {recipient}\nSender: {sender}\nPayload:\n{}",
480                message.text
481            )
482        })
483        .collect::<Vec<_>>()
484        .join("\n\n")
485}
486
487fn canonical_team_member_path(member: &TeamMemberDescriptor) -> String {
488    if let Some(agent_path) = member
489        .agent_path
490        .as_deref()
491        .filter(|path| *path == "/root" || path.starts_with("/root/"))
492    {
493        return agent_path.to_string();
494    }
495    if member.role == TeamMemberRole::Lead {
496        return "/root".to_string();
497    }
498    member
499        .task_name
500        .as_deref()
501        .filter(|name| !name.trim().is_empty())
502        .map(|name| format!("/root/{}", name.trim().trim_matches('/')))
503        .unwrap_or_else(|| format!("/root/{}", member.id))
504}
505
506fn is_codex_v2_team(team: &TeamState) -> bool {
507    team.members.iter().any(|member| {
508        member
509            .model
510            .as_deref()
511            .is_some_and(|model| model_supports_reasoning_effort(model, REASONING_ULTRA))
512    })
513}
514
515fn runtime_local_team_view(
516    mut team: TeamState,
517    active_thread_ids: &std::collections::HashSet<ThreadId>,
518) -> TeamState {
519    for member in &mut team.members {
520        if member.status == TeamMemberStatus::Running
521            && !active_thread_ids.contains(&member.thread_id)
522        {
523            member.status = TeamMemberStatus::Interrupted;
524            member.current_turn_id = None;
525        }
526    }
527    team
528}
529
530type RunnerToolExecutionKey = (String, String);
531type RunnerToolExecutionMutex = Mutex<()>;
532type RunnerToolExecutionLockRegistry =
533    Mutex<HashMap<RunnerToolExecutionKey, Weak<RunnerToolExecutionMutex>>>;
534
535pub struct Runtime {
536    pub bus: EventBus,
537    pub registry: ExtensionRegistry,
538    config: RwLock<RuntimeConfig>,
539    pub(crate) execution_lease: Option<Arc<crate::execution_lease::RuntimeExecutionLease>>,
540    pending_plan_exit: RwLock<Option<PendingPlanExit>>,
541    pub(crate) pending_tool_approvals: Mutex<HashMap<String, PendingToolApproval>>,
542    pub(crate) pending_user_inputs: Mutex<HashMap<String, PendingUserInput>>,
543    pub(crate) pending_external_tool_calls: Mutex<HashMap<String, PendingExternalToolCall>>,
544    /// Serializes turn admission with shutdown quiescing. A drain takes this
545    /// gate before flipping `accepting_turns` and snapshotting active work so a
546    /// turn that observed the old flag cannot be inserted after the snapshot.
547    turn_admission: Mutex<()>,
548    active_turns: RwLock<HashMap<TurnId, ActiveTurnHandle>>,
549    turn_drains: RwLock<HashMap<TurnId, Arc<TurnDrainHandle>>>,
550    /// Provider cleanup handles register synchronously through the inference
551    /// tool-executor context. They are intentionally separate from active turn
552    /// admission so an interrupted turn can remain drainable after a follow-up
553    /// turn starts on the same thread.
554    provider_turn_cleanups: std::sync::Mutex<HashMap<TurnId, Arc<dyn ProviderTurnCleanup>>>,
555    accepting_turns: AtomicBool,
556    /// Monotonic counter used by bounded drains to surface lifecycle-state
557    /// persistence failures without making an individual provider turn fail.
558    lifecycle_persistence_failures: AtomicUsize,
559    lifecycle_shutdown_drains: AtomicUsize,
560    lifecycle_clean_shutdowns: AtomicUsize,
561    lifecycle_deadline_exceeded: AtomicUsize,
562    lifecycle_persistence_failed_drains: AtomicUsize,
563    lifecycle_restart_reconciliations: AtomicUsize,
564    lifecycle_shutdown_drain_duration_ms_total: AtomicUsize,
565    provider_cleanup_confirmed: AtomicUsize,
566    provider_cleanup_timed_out: AtomicUsize,
567    provider_cleanup_unknown: AtomicUsize,
568    active_turns_changed: Notify,
569    active_turn_selections: RwLock<HashMap<TurnId, ModelSelectionMode>>,
570    compaction_templates: RwLock<HashMap<ThreadId, AgentInferenceRequest>>,
571    thread_admission_gates: Mutex<HashMap<ThreadId, Arc<Mutex<()>>>>,
572    /// Threads that have already dispatched a `SessionStart` local hook.
573    ///
574    /// `ThreadCreated` fires once, but `ThreadLoaded` fires on every read of the
575    /// thread from its store, so without this a session ran its setup hooks
576    /// several times — including after the turn had already stopped.
577    session_hook_started: RwLock<HashSet<ThreadId>>,
578    active_turn_contexts: RwLock<HashMap<TurnId, InheritedTurnContext>>,
579    /// Process-local execution boundary. Local/CLI runtimes default to true;
580    /// hosted runtime pools set this false so missing runner metadata fails
581    /// closed instead of exposing the host workspace.
582    allow_local_workspaces: AtomicBool,
583    // Spawn-time live authority retained for every reusable turn of a long-lived teammate.
584    // This stays process-local because developer_context is explicitly volatile and must never
585    // be persisted to thread state.
586    team_member_turn_contexts: Mutex<HashMap<ThreadId, InheritedTurnContext>>,
587    workspace: PathBuf,
588    pub(crate) teams: TeamManager,
589    agent_team_spawn_lock: Mutex<()>,
590    pub(crate) roadmaps: Mutex<roder_roadmap::RoadmapRuntime>,
591    /// Completed reviews, most recent last, so a later publish can resolve a
592    /// review id back to its findings. Bounded; see `review::run`.
593    pub(crate) reviews: Mutex<Vec<crate::review::ReviewRecord>>,
594    pub(crate) goals: Arc<RuntimeGoalController>,
595    context_artifacts: roder_api::artifacts::ContextArtifactStore,
596    pub(crate) thread_store: Option<Arc<dyn ThreadStore>>,
597    thread_item_cache: Mutex<ThreadItemCache>,
598    pub(crate) tool_registry: ToolRegistry,
599    media_generation: Arc<crate::media_generation::MediaGenerationService>,
600    pub(crate) skills: RwLock<SkillRegistry>,
601    /// Lazily-started bounded dispatch of emitted events to registry
602    /// `EventSink`s (process extensions etc.); see `event_sink_dispatch`.
603    event_sink_dispatcher: tokio::sync::OnceCell<crate::event_sink_dispatch::EventSinkDispatcher>,
604    pub(crate) compaction_hysteresis:
605        std::sync::Mutex<HashMap<ThreadId, crate::compaction_runtime::CompactionState>>,
606    /// Per-thread agent-swarm mode overrides (roadmap 104). A thread present in
607    /// this map uses its stored value; absent threads fall back to the
608    /// runtime-global `RuntimeConfig.agent_swarm_mode` default. This mirrors the
609    /// team per-member policy-mode override idiom so swarm mode is per-thread
610    /// like the other `thread/*` operations, without a separate runtime.
611    agent_swarm_modes: RwLock<HashMap<ThreadId, bool>>,
612    /// Per-thread ultra mode overrides. A thread present in this map uses its
613    /// stored value; absent threads fall back to the runtime-global
614    /// `RuntimeConfig.ultra_mode` default.
615    ultra_modes: RwLock<HashMap<ThreadId, bool>>,
616    /**
617     * Live remote-runner sessions keyed by thread. Each per-thread mutex is
618     * held across provider initialization, making the first workspace-tool
619     * access a singleflight while allowing unrelated threads to initialize in
620     * parallel. Failed initialization is shared with current waiters, then
621     * evicted so a later tool call can retry.
622     */
623    runner_sessions: Arc<Mutex<HashMap<ThreadId, Arc<RunnerSessionSlot>>>>,
624    /**
625     * Outer tool-call locks keyed by the actual remote session, not the
626     * per-thread destination id. Hosted runtimes are tenant-scoped, so this
627     * serializes threads that share one sandbox without coupling different
628     * tenants or independent runner sessions.
629     */
630    runner_tool_execution_locks: RunnerToolExecutionLockRegistry,
631}
632
633#[derive(Clone)]
634struct CachedRunnerSession {
635    destination: RunnerDestination,
636    session: Arc<dyn RemoteRunnerSession>,
637}
638
639struct RunnerSessionSlot {
640    provider_id: String,
641    destination_id: String,
642    state: Mutex<RunnerSessionSlotState>,
643    initialized: Notify,
644}
645
646enum RunnerSessionSlotState {
647    Empty,
648    Initializing,
649    Ready(CachedRunnerSession),
650    Failed(Arc<str>),
651}
652
653impl RunnerSessionSlot {
654    fn empty(destination: &RunnerDestination) -> Self {
655        Self {
656            provider_id: destination.provider_id.clone(),
657            destination_id: destination.id.clone(),
658            state: Mutex::new(RunnerSessionSlotState::Empty),
659            initialized: Notify::new(),
660        }
661    }
662
663    fn ready(session: CachedRunnerSession) -> Self {
664        Self {
665            provider_id: session.destination.provider_id.clone(),
666            destination_id: session.destination.id.clone(),
667            state: Mutex::new(RunnerSessionSlotState::Ready(session)),
668            initialized: Notify::new(),
669        }
670    }
671
672    fn matches(&self, destination: &RunnerDestination) -> bool {
673        self.provider_id == destination.provider_id && self.destination_id == destination.id
674    }
675}
676
677impl Runtime {
678    pub fn new(registry: ExtensionRegistry, config: RuntimeConfig) -> anyhow::Result<Self> {
679        if registry.inference_engines.is_empty() {
680            anyhow::bail!("at least one inference engine must be registered");
681        }
682        validate_runtime_config_reasoning(&config)?;
683        validate_runtime_inference_router_config(&registry, &config)?;
684
685        let bus = EventBus::new(EVENT_BUS_CAPACITY);
686        let thread_store = registry
687            .thread_stores
688            .first()
689            .map(|factory| factory.create());
690        let mut tool_registry = ToolRegistry::default();
691        for contributor in &registry.tools {
692            contributor
693                .contribute(&mut tool_registry)
694                .with_context(|| format!("tool contributor {} failed", contributor.id()))?;
695        }
696        crate::agent_control_tools::contribute_agent_control_tools(&mut tool_registry)?;
697
698        let media_generation = Arc::new(crate::media_generation::MediaGenerationService::new(
699            registry.media_generator_providers.clone(),
700            config.media_generation.clone(),
701        ));
702        tool_registry.replace(Arc::new(
703            crate::media_generation::MediaGenerateImageTool::new(media_generation.clone()),
704        ));
705
706        let team_data_dir = config.team_data_dir.clone();
707        let workspace = config
708            .workspace
709            .clone()
710            .map(PathBuf::from)
711            .unwrap_or(std::env::current_dir()?);
712        let roadmap_data_dir = config
713            .roadmap_data_dir
714            .clone()
715            .unwrap_or_else(|| workspace.join(".roder"));
716        let context_artifacts = thread_store
717            .as_ref()
718            .and_then(|store| store.context_artifact_store())
719            .or_else(|| {
720                thread_store
721                    .as_ref()
722                    .and_then(|store| store.local_thread_root())
723                    .map(FilesystemContextArtifactStore::shared_thread_scoped)
724            })
725            .unwrap_or_else(|| {
726                FilesystemContextArtifactStore::shared_legacy(default_context_artifact_dir())
727            });
728        let goals = Arc::new(RuntimeGoalController::new(
729            bus.clone(),
730            thread_store.clone(),
731        ));
732        let runtime = Self {
733            bus,
734            registry,
735            config: RwLock::new(config),
736            execution_lease: None,
737            pending_plan_exit: RwLock::new(None),
738            pending_tool_approvals: Mutex::new(HashMap::new()),
739            pending_user_inputs: Mutex::new(HashMap::new()),
740            pending_external_tool_calls: Mutex::new(HashMap::new()),
741            turn_admission: Mutex::new(()),
742            active_turns: RwLock::new(HashMap::new()),
743            turn_drains: RwLock::new(HashMap::new()),
744            provider_turn_cleanups: std::sync::Mutex::new(HashMap::new()),
745            accepting_turns: AtomicBool::new(true),
746            lifecycle_persistence_failures: AtomicUsize::new(0),
747            lifecycle_shutdown_drains: AtomicUsize::new(0),
748            lifecycle_clean_shutdowns: AtomicUsize::new(0),
749            lifecycle_deadline_exceeded: AtomicUsize::new(0),
750            lifecycle_persistence_failed_drains: AtomicUsize::new(0),
751            lifecycle_restart_reconciliations: AtomicUsize::new(0),
752            lifecycle_shutdown_drain_duration_ms_total: AtomicUsize::new(0),
753            provider_cleanup_confirmed: AtomicUsize::new(0),
754            provider_cleanup_timed_out: AtomicUsize::new(0),
755            provider_cleanup_unknown: AtomicUsize::new(0),
756            active_turns_changed: Notify::new(),
757            active_turn_selections: RwLock::new(HashMap::new()),
758            compaction_templates: RwLock::new(HashMap::new()),
759            thread_admission_gates: Mutex::new(HashMap::new()),
760            session_hook_started: RwLock::new(HashSet::new()),
761            active_turn_contexts: RwLock::new(HashMap::new()),
762            allow_local_workspaces: AtomicBool::new(true),
763            team_member_turn_contexts: Mutex::new(HashMap::new()),
764            workspace: workspace.clone(),
765            teams: TeamManager::new(
766                team_data_dir.unwrap_or_else(crate::teams::default_team_data_dir),
767            ),
768            agent_team_spawn_lock: Mutex::new(()),
769            roadmaps: Mutex::new(roder_roadmap::RoadmapRuntime::new(
770                workspace,
771                roadmap_data_dir,
772            )),
773            reviews: Mutex::new(Vec::new()),
774            goals,
775            context_artifacts,
776            thread_store,
777            thread_item_cache: Mutex::new(ThreadItemCache::default()),
778            tool_registry,
779            media_generation,
780            skills: RwLock::new(SkillRegistry::load(SkillRegistryOptions::new(
781                PathBuf::new(),
782            ))),
783            event_sink_dispatcher: tokio::sync::OnceCell::new(),
784            compaction_hysteresis: crate::compaction_runtime::compaction_hysteresis_state(),
785            agent_swarm_modes: RwLock::new(HashMap::new()),
786            ultra_modes: RwLock::new(HashMap::new()),
787            runner_sessions: Arc::new(Mutex::new(HashMap::new())),
788            runner_tool_execution_locks: Mutex::new(HashMap::new()),
789        };
790        runtime.bus.emit(RoderEvent::RuntimeStarted(RuntimeStarted {
791            timestamp: OffsetDateTime::now_utc(),
792        }));
793        for manifest in &runtime.registry.manifests {
794            runtime
795                .bus
796                .emit(RoderEvent::ExtensionRegistered(ExtensionRegistered {
797                    extension_id: manifest.id.clone(),
798                    timestamp: OffsetDateTime::now_utc(),
799                }));
800        }
801        Ok(runtime)
802    }
803
804    pub fn from_engine(engine: Arc<dyn InferenceEngine>) -> anyhow::Result<Self> {
805        let mut builder = roder_api::extension::ExtensionRegistryBuilder::new();
806        builder.inference_engine(engine);
807        Self::new(builder.build()?, RuntimeConfig::default())
808    }
809
810    pub fn fake() -> anyhow::Result<Self> {
811        Self::from_engine(Arc::new(FakeInferenceEngine))
812    }
813
814    pub fn subscribe_events(&self) -> tokio::sync::broadcast::Receiver<EventEnvelope> {
815        self.bus.subscribe()
816    }
817
818    /// Returns process-local, redacted lifecycle health counters. The snapshot
819    /// intentionally has fixed fields and contains no provider, command,
820    /// process, thread, or turn identifiers.
821    pub fn lifecycle_metrics(&self) -> LifecycleMetricsSnapshot {
822        LifecycleMetricsSnapshot {
823            shutdown_drain_count: self.lifecycle_shutdown_drains.load(Ordering::Acquire) as u64,
824            clean_shutdown_count: self.lifecycle_clean_shutdowns.load(Ordering::Acquire) as u64,
825            deadline_exceeded_count: self.lifecycle_deadline_exceeded.load(Ordering::Acquire)
826                as u64,
827            persistence_failed_count: self
828                .lifecycle_persistence_failed_drains
829                .load(Ordering::Acquire) as u64,
830            restart_reconciliation_count: self
831                .lifecycle_restart_reconciliations
832                .load(Ordering::Acquire) as u64,
833            lifecycle_persistence_failure_count: self
834                .lifecycle_persistence_failures
835                .load(Ordering::Acquire) as u64,
836            shutdown_drain_duration_ms_total: self
837                .lifecycle_shutdown_drain_duration_ms_total
838                .load(Ordering::Acquire) as u64,
839            provider_cleanup_confirmed_count: self
840                .provider_cleanup_confirmed
841                .load(Ordering::Acquire) as u64,
842            provider_cleanup_timed_out_count: self
843                .provider_cleanup_timed_out
844                .load(Ordering::Acquire) as u64,
845            provider_cleanup_unknown_count: self.provider_cleanup_unknown.load(Ordering::Acquire)
846                as u64,
847        }
848    }
849
850    async fn persist_turn_lifecycle_record(&self, record: &TurnLifecycleRecord) -> bool {
851        let Some(store) = &self.thread_store else {
852            return true;
853        };
854        let state = match record.extension_state() {
855            Ok(state) => state,
856            Err(_) => {
857                self.lifecycle_persistence_failures
858                    .fetch_add(1, Ordering::AcqRel);
859                return false;
860            }
861        };
862        // Lifecycle diagnostics must not turn a provider result into a failed
863        // turn merely because a third-party store lacks extension-state support.
864        // JSONL/Postgres stores persist this append-only record atomically at
865        // their own storage boundary.
866        if store
867            .append_extension_state(&record.thread_id, &state)
868            .await
869            .is_err()
870        {
871            self.lifecycle_persistence_failures
872                .fetch_add(1, Ordering::AcqRel);
873            return false;
874        }
875        true
876    }
877
878    async fn record_turn_lifecycle(
879        &self,
880        thread_id: ThreadId,
881        turn_id: TurnId,
882        state: TurnLifecycleState,
883        cleanup: TurnCleanupState,
884        reason: Option<TurnLifecycleReason>,
885    ) -> TurnLifecycleRecord {
886        self.record_turn_lifecycle_with_ownership(
887            thread_id,
888            turn_id,
889            state,
890            cleanup,
891            reason,
892            TurnCleanupOwnership::RuntimeTaskOnly,
893        )
894        .await
895    }
896
897    async fn record_turn_lifecycle_with_ownership(
898        &self,
899        thread_id: ThreadId,
900        turn_id: TurnId,
901        state: TurnLifecycleState,
902        cleanup: TurnCleanupState,
903        reason: Option<TurnLifecycleReason>,
904        ownership: TurnCleanupOwnership,
905    ) -> TurnLifecycleRecord {
906        let record = TurnLifecycleRecord::new(
907            thread_id,
908            turn_id,
909            state,
910            cleanup,
911            reason,
912            OffsetDateTime::now_utc(),
913        )
914        .with_ownership(ownership);
915        let _ = self.persist_turn_lifecycle_record(&record).await;
916        self.emit(RoderEvent::TurnLifecycleUpdated(record.clone()))
917            .await;
918        record
919    }
920
921    pub(crate) fn register_provider_turn_cleanup(
922        &self,
923        turn_id: &TurnId,
924        cleanup: Arc<dyn ProviderTurnCleanup>,
925    ) {
926        if let Ok(mut cleanups) = self.provider_turn_cleanups.lock() {
927            cleanups.insert(turn_id.clone(), cleanup);
928        }
929    }
930
931    fn provider_cleanup_ownership(&self, turn_id: &TurnId) -> TurnCleanupOwnership {
932        self.provider_turn_cleanups
933            .lock()
934            .ok()
935            .and_then(|cleanups| cleanups.get(turn_id).cloned())
936            .map(|cleanup| cleanup.ownership())
937            .unwrap_or(TurnCleanupOwnership::RuntimeTaskOnly)
938    }
939
940    async fn await_provider_turn_cleanup(
941        &self,
942        turn_id: &TurnId,
943    ) -> (TurnCleanupState, TurnCleanupOwnership) {
944        let cleanup = self
945            .provider_turn_cleanups
946            .lock()
947            .ok()
948            .and_then(|mut cleanups| cleanups.remove(turn_id));
949        let Some(cleanup) = cleanup else {
950            return (
951                TurnCleanupState::Unknown,
952                TurnCleanupOwnership::RuntimeTaskOnly,
953            );
954        };
955        let ownership = cleanup.ownership();
956        match tokio::time::timeout(PROVIDER_CLEANUP_TIMEOUT, cleanup.wait_for_cleanup()).await {
957            Ok(Ok(())) => {
958                self.provider_cleanup_confirmed
959                    .fetch_add(1, Ordering::AcqRel);
960                (
961                    TurnCleanupState::Completed,
962                    TurnCleanupOwnership::ProviderCleanupConfirmed,
963                )
964            }
965            Ok(Err(_)) => {
966                self.provider_cleanup_unknown.fetch_add(1, Ordering::AcqRel);
967                (TurnCleanupState::Unknown, ownership)
968            }
969            Err(_) => {
970                self.provider_cleanup_timed_out
971                    .fetch_add(1, Ordering::AcqRel);
972                (TurnCleanupState::TimedOut, ownership)
973            }
974        }
975    }
976
977    fn record_lifecycle_drain_outcome(
978        &self,
979        started_at: tokio::time::Instant,
980        outcome: &RuntimeDrainOutcome,
981    ) {
982        self.lifecycle_shutdown_drains
983            .fetch_add(1, Ordering::AcqRel);
984        self.lifecycle_shutdown_drain_duration_ms_total.fetch_add(
985            started_at.elapsed().as_millis().min(usize::MAX as u128) as usize,
986            Ordering::AcqRel,
987        );
988        match outcome {
989            RuntimeDrainOutcome::Clean { .. } => {
990                self.lifecycle_clean_shutdowns
991                    .fetch_add(1, Ordering::AcqRel);
992            }
993            RuntimeDrainOutcome::DeadlineExceeded { .. } => {
994                self.lifecycle_deadline_exceeded
995                    .fetch_add(1, Ordering::AcqRel);
996            }
997            RuntimeDrainOutcome::PersistenceFailed { .. } => {
998                self.lifecycle_persistence_failed_drains
999                    .fetch_add(1, Ordering::AcqRel);
1000            }
1001        }
1002    }
1003
1004    fn lifecycle_record_for_event(event: &RoderEvent) -> Option<TurnLifecycleRecord> {
1005        match event {
1006            RoderEvent::TurnStarted(event) => Some(TurnLifecycleRecord::new(
1007                event.thread_id.clone(),
1008                event.turn_id.clone(),
1009                TurnLifecycleState::Running,
1010                TurnCleanupState::NotRequested,
1011                None,
1012                event.timestamp,
1013            )),
1014            // `run_turn` persists completed turns after it has awaited a
1015            // registered provider cleanup handle. Do not let the generic event
1016            // projection overwrite that acknowledgement with an unproven state.
1017            RoderEvent::TurnCompleted(_) => None,
1018            // Some failure paths already have an event before the turn wrapper
1019            // reaches its cleanup acknowledgement. Preserve a durable fallback
1020            // reason here; the wrapper appends a later record with the more
1021            // precise cleanup outcome when a provider registered one.
1022            RoderEvent::TurnFailed(event) => Some(TurnLifecycleRecord::new(
1023                event.thread_id.clone(),
1024                event.turn_id.clone(),
1025                TurnLifecycleState::Failed,
1026                TurnCleanupState::Unknown,
1027                Some(if event.error_kind.as_deref() == Some("deadline_timeout") {
1028                    TurnLifecycleReason::DeadlineExceeded
1029                } else {
1030                    TurnLifecycleReason::ProviderFailure
1031                }),
1032                event.timestamp,
1033            )),
1034            RoderEvent::TurnDeadlineExceeded(_) | RoderEvent::TurnInterrupted(_) => None,
1035            RoderEvent::TurnLifecycleUpdated(_) => None,
1036            _ => None,
1037        }
1038    }
1039
1040    pub fn registry(&self) -> &ExtensionRegistry {
1041        &self.registry
1042    }
1043
1044    pub fn media_generation(&self) -> Arc<crate::media_generation::MediaGenerationService> {
1045        self.media_generation.clone()
1046    }
1047
1048    pub fn context_artifacts(&self) -> roder_api::artifacts::ContextArtifactStore {
1049        self.context_artifacts.clone()
1050    }
1051
1052    pub async fn execute_workflow_tool(
1053        &self,
1054        thread_id: ThreadId,
1055        tool_name: &str,
1056        arguments: serde_json::Value,
1057    ) -> anyhow::Result<ToolResult> {
1058        let Some(executor) = self.tool_registry.get(tool_name) else {
1059            anyhow::bail!("tool not found: {tool_name}");
1060        };
1061        let tool_call = ToolCall {
1062            id: format!("slash-{tool_name}"),
1063            name: tool_name.to_string(),
1064            raw_arguments: serde_json::to_string(&arguments)?,
1065            arguments,
1066            thread_id: thread_id.clone(),
1067            turn_id: "slash-command".to_string(),
1068        };
1069        let runtime_config = self.status().await;
1070        let ctx = self.tool_execution_context(
1071            thread_id,
1072            "slash-command".to_string(),
1073            runtime_config.policy_mode,
1074            runtime_config.workspace.as_deref(),
1075            Some(&runtime_config.command_shell),
1076        );
1077        executor.execute(ctx, tool_call).await
1078    }
1079
1080    pub(crate) fn tool_execution_context(
1081        &self,
1082        thread_id: ThreadId,
1083        turn_id: TurnId,
1084        mode: PolicyMode,
1085        workspace: Option<&str>,
1086        command_shell: Option<&str>,
1087    ) -> ToolExecutionContext {
1088        let mut ctx = ToolExecutionContext::new(thread_id, turn_id, mode)
1089            .with_command_shell(command_shell.unwrap_or_default())
1090            .with_process_runner(Arc::new(LocalProcessRunner))
1091            .with_context_artifacts(self.context_artifacts.backend())
1092            .with_goal_controller(self.goals.clone())
1093            .with_subagent_trace_sink(Arc::new(RuntimeSubagentTraceSink::new(
1094                self.bus.clone(),
1095                self.thread_store.clone(),
1096            )))
1097            .with_swarm_progress_sink(Arc::new(RuntimeAgentSwarmProgressSink::new(
1098                self.bus.clone(),
1099                self.thread_store.clone(),
1100            )));
1101        if let Some(workspace) = workspace {
1102            ctx = ctx.with_workspace_handle(Arc::new(ScopedFilesystem::new(workspace)));
1103        }
1104        ctx
1105    }
1106
1107    pub async fn status(&self) -> RuntimeConfig {
1108        self.config.read().await.clone()
1109    }
1110
1111    pub async fn set_skills(&self, skills: SkillRegistry) {
1112        *self.skills.write().await = skills;
1113    }
1114
1115    pub async fn skills_snapshot(&self) -> SkillRegistry {
1116        self.skills.read().await.clone()
1117    }
1118
1119    pub fn workspace(&self) -> PathBuf {
1120        self.workspace.clone()
1121    }
1122
1123    /// Controls whether native workspace tools may execute without an
1124    /// explicit remote-runner binding. Hosted runtimes disable this after
1125    /// construction; ordinary local runtimes retain the compatible default.
1126    pub fn set_allow_local_workspaces(&self, allowed: bool) {
1127        self.allow_local_workspaces
1128            .store(allowed, Ordering::Relaxed);
1129    }
1130
1131    pub fn allows_local_workspaces(&self) -> bool {
1132        self.allow_local_workspaces.load(Ordering::Relaxed)
1133    }
1134
1135    pub async fn set_remote_runner_destination(&self, destination: Option<RunnerDestination>) {
1136        let lifecycle = destination.as_ref().map(|destination| RunnerLifecycle {
1137            destination_id: destination.id.clone(),
1138            provider_id: destination.provider_id.clone(),
1139            state: "configured".to_string(),
1140            session_id: None,
1141            timestamp: OffsetDateTime::now_utc(),
1142        });
1143        self.config.write().await.remote_runner_destination = destination;
1144        if let Some(lifecycle) = lifecycle {
1145            self.emit(RoderEvent::RunnerLifecycle(lifecycle)).await;
1146        } else {
1147            self.emit(RoderEvent::RunnerLifecycle(RunnerLifecycle {
1148                destination_id: "local".to_string(),
1149                provider_id: "local".to_string(),
1150                state: "local_fallback".to_string(),
1151                session_id: None,
1152                timestamp: OffsetDateTime::now_utc(),
1153            }))
1154            .await;
1155        }
1156    }
1157
1158    pub async fn set_file_backed_dynamic_context(&self, enabled: bool) -> RuntimeConfig {
1159        let mut cfg = self.config.write().await;
1160        cfg.file_backed_dynamic_context = enabled;
1161        cfg.clone()
1162    }
1163
1164    pub async fn set_command_shell(&self, shell: String) -> RuntimeConfig {
1165        let mut cfg = self.config.write().await;
1166        cfg.command_shell = shell;
1167        cfg.clone()
1168    }
1169
1170    pub async fn pending_plan_exit(&self) -> Option<PendingPlanExit> {
1171        let mut pending = self.pending_plan_exit.write().await;
1172        let current = pending.clone()?;
1173        if !current.is_expired(OffsetDateTime::now_utc()) {
1174            return Some(current);
1175        }
1176        *pending = None;
1177        drop(pending);
1178        self.emit_plan_exit_resolved(&current, false, self.status().await.policy_mode)
1179            .await;
1180        None
1181    }
1182
1183    pub async fn set_policy_mode(
1184        &self,
1185        mode: PolicyMode,
1186        reason: Option<String>,
1187    ) -> anyhow::Result<RuntimeConfig> {
1188        let mut cfg = self.config.write().await;
1189        let previous_mode = cfg.policy_mode;
1190        cfg.policy_mode = mode;
1191        let next = cfg.clone();
1192        drop(cfg);
1193        self.emit(RoderEvent::PolicyModeChanged(PolicyModeChanged {
1194            thread_id: "runtime".to_string(),
1195            turn_id: None,
1196            previous_mode,
1197            new_mode: mode,
1198            reason,
1199            timestamp: OffsetDateTime::now_utc(),
1200        }))
1201        .await;
1202        self.auto_resolve_pending_tool_approvals().await;
1203        Ok(next)
1204    }
1205
1206    /// Toggle agent-swarm mode for the runtime (roadmap 104). When enabled, the
1207    /// swarm reminder is injected into each turn's developer instructions so
1208    /// every app-server/SDK client gets it, not only the TUI.
1209    pub async fn set_agent_swarm_mode(
1210        &self,
1211        enabled: bool,
1212        trigger: roder_api::subagents::AgentSwarmModeTrigger,
1213    ) -> anyhow::Result<RuntimeConfig> {
1214        let mut cfg = self.config.write().await;
1215        cfg.agent_swarm_mode = enabled;
1216        let next = cfg.clone();
1217        drop(cfg);
1218        self.emit(RoderEvent::AgentSwarmModeChanged(
1219            roder_api::subagents::AgentSwarmModeChanged {
1220                thread_id: "runtime".to_string(),
1221                turn_id: None,
1222                enabled,
1223                trigger,
1224                timestamp: OffsetDateTime::now_utc(),
1225            },
1226        ))
1227        .await;
1228        Ok(next)
1229    }
1230
1231    /// Toggle agent-swarm mode for a single thread (roadmap 104). The override is
1232    /// stored per thread so toggling swarm mode on one thread does not leak the
1233    /// reminder into other threads sharing the runtime; absent threads fall back
1234    /// to the runtime-global default. The emitted `AgentSwarmModeChanged` event
1235    /// carries the real `thread_id`.
1236    pub async fn set_agent_swarm_mode_for_thread(
1237        &self,
1238        thread_id: &str,
1239        enabled: bool,
1240        trigger: roder_api::subagents::AgentSwarmModeTrigger,
1241    ) -> bool {
1242        {
1243            let mut modes = self.agent_swarm_modes.write().await;
1244            modes.insert(thread_id.to_string(), enabled);
1245        }
1246        self.emit(RoderEvent::AgentSwarmModeChanged(
1247            roder_api::subagents::AgentSwarmModeChanged {
1248                thread_id: thread_id.to_string(),
1249                turn_id: None,
1250                enabled,
1251                trigger,
1252                timestamp: OffsetDateTime::now_utc(),
1253            },
1254        ))
1255        .await;
1256        enabled
1257    }
1258
1259    /// Resolve whether agent-swarm mode is active for `thread_id`: the per-thread
1260    /// override when present, otherwise the runtime-global default. The turn loop
1261    /// uses this to decide whether to inject the swarm reminder.
1262    pub async fn effective_agent_swarm_mode_for_thread(&self, thread_id: &str) -> bool {
1263        if let Some(enabled) = self.agent_swarm_modes.read().await.get(thread_id).copied() {
1264            return enabled;
1265        }
1266        self.status().await.agent_swarm_mode
1267    }
1268
1269    /// Toggle ultra mode for the runtime. When enabled, proactive multi-agent
1270    /// policy is injected into each turn's developer instructions for any
1271    /// model, so every app-server/SDK client gets it (not only the TUI).
1272    pub async fn set_ultra_mode(
1273        &self,
1274        enabled: bool,
1275        trigger: roder_api::subagents::UltraModeTrigger,
1276    ) -> anyhow::Result<RuntimeConfig> {
1277        let mut cfg = self.config.write().await;
1278        cfg.ultra_mode = enabled;
1279        let next = cfg.clone();
1280        drop(cfg);
1281        self.emit(RoderEvent::UltraModeChanged(
1282            roder_api::subagents::UltraModeChanged {
1283                thread_id: "runtime".to_string(),
1284                turn_id: None,
1285                enabled,
1286                trigger,
1287                timestamp: OffsetDateTime::now_utc(),
1288            },
1289        ))
1290        .await;
1291        Ok(next)
1292    }
1293
1294    /// Toggle ultra mode for a single thread. The override is stored per thread
1295    /// so toggling on one thread does not leak proactive multi-agent policy into
1296    /// other threads sharing the runtime; absent threads fall back to the
1297    /// runtime-global default.
1298    pub async fn set_ultra_mode_for_thread(
1299        &self,
1300        thread_id: &str,
1301        enabled: bool,
1302        trigger: roder_api::subagents::UltraModeTrigger,
1303    ) -> bool {
1304        {
1305            let mut modes = self.ultra_modes.write().await;
1306            modes.insert(thread_id.to_string(), enabled);
1307        }
1308        self.emit(RoderEvent::UltraModeChanged(
1309            roder_api::subagents::UltraModeChanged {
1310                thread_id: thread_id.to_string(),
1311                turn_id: None,
1312                enabled,
1313                trigger,
1314                timestamp: OffsetDateTime::now_utc(),
1315            },
1316        ))
1317        .await;
1318        enabled
1319    }
1320
1321    /// Resolve whether ultra mode is active for `thread_id`: the per-thread
1322    /// override when present, otherwise the runtime-global default. The turn
1323    /// loop uses this (together with Sol/Terra Ultra effort) to decide whether
1324    /// to inject proactive multi-agent policy.
1325    pub async fn effective_ultra_mode_for_thread(&self, thread_id: &str) -> bool {
1326        if let Some(enabled) = self.ultra_modes.read().await.get(thread_id).copied() {
1327            return enabled;
1328        }
1329        self.status().await.ultra_mode
1330    }
1331
1332    pub async fn set_hosted_web_search(
1333        &self,
1334        mode: HostedWebSearchMode,
1335    ) -> anyhow::Result<RuntimeConfig> {
1336        let mut cfg = self.config.write().await;
1337        cfg.hosted_web_search = HostedWebSearchConfig { mode };
1338        Ok(cfg.clone())
1339    }
1340
1341    pub async fn record_pending_plan_exit(&self, pending: PendingPlanExit) {
1342        *self.pending_plan_exit.write().await = Some(pending.clone());
1343        self.emit(RoderEvent::PolicyExitPlanRequested(
1344            PolicyExitPlanRequested {
1345                thread_id: pending.thread_id,
1346                turn_id: pending.turn_id,
1347                request_id: pending.request_id,
1348                target_mode: pending.target_mode,
1349                plan_summary: pending.plan_summary,
1350                next_steps: pending.next_steps,
1351                timestamp: OffsetDateTime::now_utc(),
1352            },
1353        ))
1354        .await;
1355    }
1356
1357    pub async fn resolve_pending_plan_exit(
1358        &self,
1359        request_id: &str,
1360        approved: bool,
1361    ) -> anyhow::Result<Option<PendingPlanExit>> {
1362        let mut pending = self.pending_plan_exit.write().await;
1363        let Some(current) = pending.clone() else {
1364            return Ok(None);
1365        };
1366        if current.request_id != request_id {
1367            anyhow::bail!("pending plan exit request {request_id:?} was not found");
1368        }
1369        *pending = None;
1370        drop(pending);
1371
1372        let approved = approved && !current.is_expired(OffsetDateTime::now_utc());
1373        let resolved_mode = if approved {
1374            let mut cfg = self.config.write().await;
1375            let previous_mode = cfg.policy_mode;
1376            cfg.policy_mode = current.target_mode;
1377            drop(cfg);
1378            self.emit(RoderEvent::PolicyModeChanged(PolicyModeChanged {
1379                thread_id: current.thread_id.clone(),
1380                turn_id: Some(current.turn_id.clone()),
1381                previous_mode,
1382                new_mode: current.target_mode,
1383                reason: Some("approved plan exit".to_string()),
1384                timestamp: OffsetDateTime::now_utc(),
1385            }))
1386            .await;
1387            self.auto_resolve_pending_tool_approvals().await;
1388            current.target_mode
1389        } else {
1390            self.status().await.policy_mode
1391        };
1392        self.emit_plan_exit_resolved(&current, approved, resolved_mode)
1393            .await;
1394        Ok(Some(current))
1395    }
1396
1397    pub async fn resolve_tool_approval(
1398        &self,
1399        approval_id: &str,
1400        approved: bool,
1401    ) -> anyhow::Result<bool> {
1402        let pending = self.pending_tool_approvals.lock().await.remove(approval_id);
1403        let Some(pending) = pending else {
1404            return Ok(false);
1405        };
1406        self.emit(RoderEvent::ApprovalResolved(ApprovalResolved {
1407            thread_id: pending.thread_id,
1408            turn_id: pending.turn_id,
1409            approval_id: approval_id.to_string(),
1410            tool_id: pending.tool_id,
1411            tool_name: pending.tool_name,
1412            approved,
1413            timestamp: OffsetDateTime::now_utc(),
1414        }))
1415        .await;
1416        let _ = pending.tx.send(approved);
1417        Ok(true)
1418    }
1419
1420    /// Completes a pending host-executed tool call (`tools/resolve`). Returns false when the
1421    /// request id is unknown, already resolved, timed out, or cancelled by a turn interrupt.
1422    pub async fn resolve_external_tool_call(
1423        &self,
1424        request_id: &str,
1425        resolution: ExternalToolResolution,
1426    ) -> anyhow::Result<bool> {
1427        let mut calls = self.pending_external_tool_calls.lock().await;
1428        let Some(pending) = calls.get(request_id) else {
1429            return Ok(false);
1430        };
1431        self.emit_external_execution(RoderEvent::ExternalToolCallResolved(
1432            ExternalToolCallResolved {
1433                thread_id: pending.thread_id.clone(),
1434                turn_id: pending.turn_id.clone(),
1435                request_id: request_id.to_string(),
1436                tool_id: pending.tool_id.clone(),
1437                tool_name: pending.tool_name.clone(),
1438                outcome: ExternalToolCallOutcome::Resolved,
1439                is_error: resolution.is_error,
1440                timestamp: OffsetDateTime::now_utc(),
1441            },
1442        ))
1443        .await?;
1444        let Some(pending) = calls.remove(request_id) else {
1445            return Ok(false);
1446        };
1447        drop(calls);
1448        let _ = pending.tx.send(resolution);
1449        Ok(true)
1450    }
1451
1452    /// Drops every pending external tool call for the turn and reports them as cancelled so an
1453    /// interrupt does not leak requests waiting on `tools/resolve`.
1454    async fn cancel_pending_external_tool_calls_for_turn(&self, turn_id: &TurnId) {
1455        let cancelled = {
1456            let mut pending = self.pending_external_tool_calls.lock().await;
1457            let request_ids = pending
1458                .iter()
1459                .filter(|(_, call)| &call.turn_id == turn_id)
1460                .map(|(request_id, _)| request_id.clone())
1461                .collect::<Vec<_>>();
1462            request_ids
1463                .into_iter()
1464                .filter_map(|request_id| pending.remove(&request_id).map(|call| (request_id, call)))
1465                .collect::<Vec<_>>()
1466        };
1467        for (request_id, call) in cancelled {
1468            self.emit(RoderEvent::ExternalToolCallResolved(
1469                ExternalToolCallResolved {
1470                    thread_id: call.thread_id,
1471                    turn_id: call.turn_id,
1472                    request_id,
1473                    tool_id: call.tool_id,
1474                    tool_name: call.tool_name,
1475                    outcome: ExternalToolCallOutcome::Cancelled,
1476                    is_error: true,
1477                    timestamp: OffsetDateTime::now_utc(),
1478                },
1479            ))
1480            .await;
1481        }
1482    }
1483
1484    pub async fn resolve_user_input(
1485        &self,
1486        request_id: &str,
1487        answers: serde_json::Value,
1488    ) -> anyhow::Result<bool> {
1489        let pending = self.pending_user_inputs.lock().await.remove(request_id);
1490        let Some(pending) = pending else {
1491            return Ok(false);
1492        };
1493        self.emit(RoderEvent::UserInputResolved(UserInputResolved {
1494            thread_id: pending.thread_id,
1495            turn_id: pending.turn_id,
1496            request_id: request_id.to_string(),
1497            answers: answers.clone(),
1498            timestamp: OffsetDateTime::now_utc(),
1499        }))
1500        .await;
1501        let _ = pending.tx.send(answers);
1502        Ok(true)
1503    }
1504
1505    async fn emit_plan_exit_resolved(
1506        &self,
1507        current: &PendingPlanExit,
1508        approved: bool,
1509        resolved_mode: PolicyMode,
1510    ) {
1511        self.emit(RoderEvent::PolicyExitPlanResolved(PolicyExitPlanResolved {
1512            thread_id: current.thread_id.clone(),
1513            turn_id: current.turn_id.clone(),
1514            request_id: current.request_id.clone(),
1515            approved,
1516            target_mode: current.target_mode,
1517            resolved_mode,
1518            timestamp: OffsetDateTime::now_utc(),
1519        }))
1520        .await;
1521    }
1522
1523    pub async fn select_provider(
1524        &self,
1525        provider: String,
1526        model: Option<String>,
1527        reasoning: Option<String>,
1528    ) -> anyhow::Result<RuntimeConfig> {
1529        let next = self
1530            .preview_provider_selection(provider, model, reasoning)
1531            .await?;
1532        let mut cfg = self.config.write().await;
1533        *cfg = next;
1534        Ok(cfg.clone())
1535    }
1536
1537    pub async fn preview_provider_selection(
1538        &self,
1539        provider: String,
1540        model: Option<String>,
1541        reasoning: Option<String>,
1542    ) -> anyhow::Result<RuntimeConfig> {
1543        self.engine_for(&provider)?;
1544        let mut cfg = self.config.read().await.clone();
1545        cfg.default_provider = provider;
1546        if let Some(model) = model {
1547            cfg.default_model = model;
1548        }
1549        if let Some(reasoning) = reasoning {
1550            if reasoning == REASONING_NONE
1551                && !model_supports_reasoning(&cfg.default_model, &reasoning)
1552            {
1553                return Ok(cfg.clone());
1554            }
1555            validate_reasoning_effort(&cfg.default_model, &reasoning)?;
1556            cfg.reasoning = Some(reasoning);
1557        }
1558        Ok(cfg)
1559    }
1560
1561    pub async fn effective_reasoning(&self) -> String {
1562        let cfg = self.config.read().await;
1563        effective_reasoning_for_model(&cfg, &cfg.default_model)
1564    }
1565
1566    pub fn effective_reasoning_for_config(cfg: &RuntimeConfig) -> String {
1567        effective_reasoning_for_model(cfg, &cfg.default_model)
1568    }
1569
1570    pub async fn set_dynamic_workflow_effort(
1571        &self,
1572        effort_profile: DynamicWorkflowEffortProfile,
1573    ) -> RuntimeConfig {
1574        let mut cfg = self.config.write().await;
1575        cfg.dynamic_workflows.effort_profile = effort_profile;
1576        cfg.clone()
1577    }
1578
1579    pub async fn dynamic_workflow_trigger_decision(
1580        &self,
1581        message: &str,
1582    ) -> WorkflowTriggerDecision {
1583        let cfg = self.config.read().await;
1584        classify_workflow_trigger(message, &cfg.dynamic_workflows)
1585    }
1586
1587    pub async fn create_thread(&self, title: Option<String>) -> anyhow::Result<ThreadMetadata> {
1588        self.create_thread_with(CreateThreadRequest {
1589            title,
1590            workspace: self.workspace.display().to_string(),
1591            workspace_id: None,
1592            root_id: None,
1593            provider: None,
1594            model: None,
1595            selection_mode: None,
1596            tool_allowlist: Vec::new(),
1597            developer_instructions: None,
1598            external_tools: Vec::new(),
1599            runner: None,
1600        })
1601        .await
1602    }
1603
1604    /**
1605     * Resolves an explicit thread-runner selection into a persisted binding.
1606     * Fails fast at thread creation when the provider is missing or rejects
1607     * the destination, instead of surfacing the error on the first tool call.
1608     */
1609    async fn resolve_thread_runner_binding(
1610        &self,
1611        thread_id: &str,
1612        selection: ThreadRunnerSelection,
1613    ) -> anyhow::Result<ThreadRunnerBinding> {
1614        let provider = self
1615            .registry
1616            .remote_runner_providers
1617            .iter()
1618            .find(|provider| provider.id() == selection.provider_id)
1619            .cloned()
1620            .ok_or_else(|| {
1621                anyhow::anyhow!(
1622                    "remote runner provider {:?} is not installed",
1623                    selection.provider_id
1624                )
1625            })?;
1626        let workspace = selection.workspace.trim();
1627        anyhow::ensure!(
1628            std::path::Path::new(workspace).is_absolute(),
1629            "runner workspace must be an absolute path on the runner: {workspace:?}"
1630        );
1631        let mut read_roots = Vec::with_capacity(selection.read_roots.len());
1632        for read_root in &selection.read_roots {
1633            let trimmed = read_root.trim();
1634            anyhow::ensure!(
1635                std::path::Path::new(trimmed).is_absolute(),
1636                "runner read root must be an absolute path on the runner: {trimmed:?}"
1637            );
1638            read_roots.push(PathBuf::from(trimmed));
1639        }
1640        let destination = RunnerDestination {
1641            id: format!("thread-{thread_id}"),
1642            provider_id: selection.provider_id,
1643            config: selection.config,
1644            default_manifest: roder_api::remote_runner::RunnerManifest::default(),
1645        };
1646        provider.validate_destination(&destination).await?;
1647        Ok(ThreadRunnerBinding {
1648            destination,
1649            workspace: PathBuf::from(workspace),
1650            read_roots,
1651        })
1652    }
1653
1654    /**
1655     * Build a per-thread binding from a runtime-level destination (selected via
1656     * the TUI runner picker or config `default_destination`) when the
1657     * destination's provider advertises a default workspace. Returns `None` when
1658     * there is no destination or the provider opts out (no default workspace),
1659     * preserving the legacy behavior where such threads keep local tools.
1660     */
1661    async fn synthesize_runtime_runner_binding(
1662        &self,
1663        thread_id: &str,
1664        destination: Option<RunnerDestination>,
1665    ) -> anyhow::Result<Option<ThreadRunnerBinding>> {
1666        let Some(destination) = destination else {
1667            return Ok(None);
1668        };
1669        let Some(provider) = self
1670            .registry
1671            .remote_runner_providers
1672            .iter()
1673            .find(|provider| provider.id() == destination.provider_id)
1674        else {
1675            return Ok(None);
1676        };
1677        // An explicit workspace in the destination config wins over the
1678        // provider default; absence of both means the provider opts out.
1679        let workspace = destination
1680            .config
1681            .get("working_dir")
1682            .and_then(serde_json::Value::as_str)
1683            .map(str::to_string)
1684            .or_else(|| provider.default_workspace());
1685        let Some(workspace) = workspace else {
1686            return Ok(None);
1687        };
1688        let selection = ThreadRunnerSelection {
1689            provider_id: destination.provider_id.clone(),
1690            config: destination.config.clone(),
1691            workspace,
1692            read_roots: Vec::new(),
1693        };
1694        Ok(Some(
1695            self.resolve_thread_runner_binding(thread_id, selection)
1696                .await?,
1697        ))
1698    }
1699
1700    /// Validates a runner selection without creating a thread; the placeholder destination id only appears in error messages.
1701    pub async fn validate_thread_runner_selection(
1702        &self,
1703        selection: ThreadRunnerSelection,
1704    ) -> anyhow::Result<()> {
1705        self.resolve_thread_runner_binding("validate", selection)
1706            .await
1707            .map(|_| ())
1708    }
1709
1710    pub async fn create_thread_with(
1711        &self,
1712        req: CreateThreadRequest,
1713    ) -> anyhow::Result<ThreadMetadata> {
1714        tool_advertisement::validate_external_tool_names(&req.external_tools)?;
1715        let cfg = self.config.read().await.clone();
1716        let now = OffsetDateTime::now_utc();
1717        let workspace = validate_thread_workspace(&req.workspace)?;
1718        let provider = req.provider.unwrap_or(cfg.default_provider);
1719        let model = req.model.unwrap_or(cfg.default_model);
1720        let selection_mode = req
1721            .selection_mode
1722            .unwrap_or_else(|| ModelSelectionMode::manual(provider.clone(), model.clone(), None));
1723        let thread_id = uuid::Uuid::new_v4().to_string();
1724        let runner_binding = match req.runner {
1725            Some(selection) => Some(
1726                self.resolve_thread_runner_binding(&thread_id, selection)
1727                    .await?,
1728            ),
1729            // No explicit per-thread selection: if a runtime-level destination
1730            // is active and its provider advertises a default workspace, bind
1731            // the new thread so its coding tools route into the runner. This is
1732            // what makes a TUI/config-selected runner (e.g. Blaxel) actually
1733            // execute tools in the sandbox instead of locally. Providers that
1734            // return no default workspace stay unbound (legacy local-tools).
1735            None => {
1736                self.synthesize_runtime_runner_binding(
1737                    &thread_id,
1738                    cfg.remote_runner_destination.clone(),
1739                )
1740                .await?
1741            }
1742        };
1743        let runner_destination = runner_binding
1744            .as_ref()
1745            .map(|binding| binding.destination.clone())
1746            .or_else(|| cfg.remote_runner_destination.clone());
1747        let metadata = ThreadMetadata {
1748            thread_id,
1749            title: req.title,
1750            workspace,
1751            workspace_id: req.workspace_id,
1752            root_id: req.root_id,
1753            provider: Some(provider),
1754            model: Some(model),
1755            selection_mode: Some(selection_mode),
1756            tool_allowlist: req.tool_allowlist,
1757            developer_instructions: req.developer_instructions,
1758            external_tools: req.external_tools,
1759            runner_destination,
1760            runner_state: None,
1761            runner_binding,
1762            created_at: now,
1763            updated_at: now,
1764            message_count: 0,
1765            usage: None,
1766            parent_thread_id: None,
1767            forked_from_turn_id: None,
1768            workspace_fork: None,
1769        };
1770
1771        let metadata = if let Some(store) = &self.thread_store {
1772            store.create_thread(metadata).await?
1773        } else {
1774            metadata
1775        };
1776        self.emit(RoderEvent::ThreadCreated(ThreadCreated {
1777            thread_id: metadata.thread_id.clone(),
1778            timestamp: OffsetDateTime::now_utc(),
1779        }))
1780        .await;
1781        Ok(metadata)
1782    }
1783
1784    pub async fn list_threads(&self) -> anyhow::Result<Vec<ThreadMetadata>> {
1785        if let Some(store) = &self.thread_store {
1786            return store.list_threads().await;
1787        }
1788        Ok(Vec::new())
1789    }
1790
1791    pub async fn list_threads_page(
1792        &self,
1793        options: roder_api::thread::ThreadListOptions,
1794    ) -> anyhow::Result<roder_api::thread::ThreadListPage> {
1795        if let Some(store) = &self.thread_store {
1796            return store.list_threads_page(options).await;
1797        }
1798        Ok(roder_api::thread::ThreadListPage::default())
1799    }
1800
1801    pub async fn load_thread_metadata(
1802        &self,
1803        thread_id: &str,
1804    ) -> anyhow::Result<Option<roder_api::thread::ThreadMetadata>> {
1805        if let Some(store) = &self.thread_store {
1806            return store.load_thread_metadata(&thread_id.to_string()).await;
1807        }
1808        Ok(None)
1809    }
1810
1811    pub async fn archive_thread(&self, thread_id: &str) -> anyhow::Result<bool> {
1812        let archived = if let Some(store) = &self.thread_store {
1813            store.archive_thread(&thread_id.to_string()).await?
1814        } else {
1815            false
1816        };
1817        if archived {
1818            let workspace = self.config.read().await.workspace.clone();
1819            crate::hooks::run_lifecycle(
1820                self,
1821                &thread_id.to_string(),
1822                &SESSION_HOOK_TURN_ID.to_string(),
1823                workspace.as_deref(),
1824                "SessionEnd",
1825                Some("clear"),
1826                serde_json::json!({"reason": "archive"}),
1827            )
1828            .await;
1829            self.thread_item_cache
1830                .lock()
1831                .await
1832                .remove_thread(&thread_id.to_string());
1833            self.evict_runner_session(&thread_id.to_string()).await;
1834        }
1835        Ok(archived)
1836    }
1837
1838    pub async fn start_team(&self, req: TeamStartRequest) -> anyhow::Result<TeamState> {
1839        let cfg = self.config.read().await.clone();
1840        let workspace = self.workspace.display().to_string();
1841        let lead_thread_id = match req.lead_thread_id {
1842            Some(thread_id) => thread_id,
1843            None => {
1844                self.create_thread_with(CreateThreadRequest {
1845                    title: Some("Team lead".to_string()),
1846                    workspace: workspace.clone(),
1847                    workspace_id: None,
1848                    root_id: None,
1849                    provider: None,
1850                    model: None,
1851                    selection_mode: None,
1852                    tool_allowlist: Vec::new(),
1853                    developer_instructions: None,
1854                    external_tools: Vec::new(),
1855                    runner: None,
1856                })
1857                .await?
1858                .thread_id
1859            }
1860        };
1861        let team_id = uuid::Uuid::new_v4().to_string();
1862        let active_lead_turn_id = self.active_turn_for_thread(&lead_thread_id).await;
1863        let lead_selection = match active_lead_turn_id.as_ref() {
1864            Some(turn_id) => self
1865                .active_turn_selections
1866                .read()
1867                .await
1868                .get(turn_id)
1869                .cloned(),
1870            None => self.selection_mode_for_thread(&lead_thread_id).await?,
1871        };
1872        let lead_concrete_selection = lead_selection
1873            .as_ref()
1874            .map(ModelSelectionMode::concrete_selection);
1875        let mut lead = crate::teams::lead_member(
1876            lead_thread_id.clone(),
1877            lead_concrete_selection
1878                .as_ref()
1879                .map(|selection| selection.provider.clone())
1880                .or_else(|| Some(cfg.default_provider.clone())),
1881            lead_concrete_selection
1882                .as_ref()
1883                .map(|selection| selection.model.clone())
1884                .or_else(|| Some(cfg.default_model.clone())),
1885            cfg.policy_mode,
1886        );
1887        if let Some(turn_id) = active_lead_turn_id {
1888            lead.current_turn_id = Some(turn_id);
1889            lead.status = TeamMemberStatus::Running;
1890        }
1891        let mut members = vec![lead];
1892
1893        for (index, member) in req.members.into_iter().enumerate() {
1894            let thread = self
1895                .create_thread_with(CreateThreadRequest {
1896                    title: Some(member.name.clone()),
1897                    workspace: workspace.clone(),
1898                    workspace_id: None,
1899                    root_id: None,
1900                    provider: member.model_provider.clone(),
1901                    model: member.model.clone(),
1902                    selection_mode: None,
1903                    tool_allowlist: Vec::new(),
1904                    developer_instructions: None,
1905                    external_tools: Vec::new(),
1906                    runner: None,
1907                })
1908                .await?;
1909            let member_id = format!("member-{}", index + 1);
1910            let descriptor = crate::teams::teammate_member(
1911                member_id.clone(),
1912                member.name,
1913                thread.thread_id.clone(),
1914                member.model_provider.or(thread.provider),
1915                member.model.or(thread.model),
1916                cfg.policy_mode,
1917            );
1918            members.push(descriptor);
1919        }
1920
1921        let now = OffsetDateTime::now_utc();
1922        let team = self
1923            .teams
1924            .insert(TeamState {
1925                id: team_id.clone(),
1926                lead_thread_id: lead_thread_id.clone(),
1927                display_mode: req.display_mode,
1928                members,
1929                mailbox: Vec::new(),
1930                tasks: Vec::new(),
1931                created_at: now,
1932                updated_at: now,
1933            })
1934            .await?;
1935        self.emit(RoderEvent::TeamStarted(TeamStarted {
1936            team_id: team_id.clone(),
1937            lead_thread_id,
1938            display_mode: team.display_mode,
1939            members: team.members.clone(),
1940            tasks: team.tasks.clone(),
1941            timestamp: OffsetDateTime::now_utc(),
1942        }))
1943        .await;
1944        for member in team
1945            .members
1946            .iter()
1947            .filter(|member| member.role != roder_api::teams::TeamMemberRole::Lead)
1948        {
1949            self.emit(RoderEvent::TeamMemberStarted(TeamMemberStarted {
1950                team_id: team_id.clone(),
1951                member: member.clone(),
1952                timestamp: OffsetDateTime::now_utc(),
1953            }))
1954            .await;
1955        }
1956        Ok(team)
1957    }
1958
1959    pub async fn list_teams(&self) -> Vec<TeamState> {
1960        let active_thread_ids = self
1961            .active_turns
1962            .read()
1963            .await
1964            .values()
1965            .map(|handle| handle.thread_id.clone())
1966            .collect::<std::collections::HashSet<_>>();
1967        self.teams
1968            .list()
1969            .await
1970            .into_iter()
1971            .map(|team| runtime_local_team_view(team, &active_thread_ids))
1972            .collect()
1973    }
1974
1975    pub async fn read_team(&self, team_id: &str) -> Option<TeamState> {
1976        let active_thread_ids = self
1977            .active_turns
1978            .read()
1979            .await
1980            .values()
1981            .map(|handle| handle.thread_id.clone())
1982            .collect::<std::collections::HashSet<_>>();
1983        self.teams
1984            .get(team_id)
1985            .await
1986            .map(|team| runtime_local_team_view(team, &active_thread_ids))
1987    }
1988
1989    pub async fn start_team_member(
1990        &self,
1991        team_id: &str,
1992        req: TeamMemberStartRequest,
1993    ) -> anyhow::Result<TeamState> {
1994        self.start_team_member_with_selection(team_id, req, None)
1995            .await
1996    }
1997
1998    pub async fn message_team_member(
1999        self: &Arc<Self>,
2000        team_id: &str,
2001        member_id: &str,
2002        message: String,
2003    ) -> anyhow::Result<TurnId> {
2004        let team = self
2005            .read_team(team_id)
2006            .await
2007            .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2008        self.followup_team_member(&team.lead_thread_id, team_id, member_id, message)
2009            .await
2010    }
2011
2012    /// Queue a mailbox message without starting an idle agent. If the target is
2013    /// already running, the message is delivered at the next inference boundary.
2014    pub(crate) async fn queue_team_member_message(
2015        self: &Arc<Self>,
2016        caller_thread_id: &ThreadId,
2017        team_id: &str,
2018        member_id: &str,
2019        message: String,
2020    ) -> anyhow::Result<Option<TurnId>> {
2021        let _delivery_guard = self.agent_team_spawn_lock.lock().await;
2022        let team = self
2023            .read_team(team_id)
2024            .await
2025            .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2026        let member = team
2027            .members
2028            .iter()
2029            .find(|member| member.id == member_id)
2030            .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2031            .clone();
2032        if member.status == TeamMemberStatus::Closed {
2033            anyhow::bail!("subagent {} is closed", member.name);
2034        }
2035        let from_member_id = team
2036            .members
2037            .iter()
2038            .find(|candidate| candidate.thread_id == *caller_thread_id)
2039            .map(|candidate| candidate.id.clone());
2040        self.teams
2041            .append_mailbox_message(
2042                team_id,
2043                from_member_id,
2044                member_id.to_string(),
2045                TeamMailboxMessageKind::Message,
2046                message,
2047            )
2048            .await?;
2049        let Some(turn_id) = self.active_turn_for_thread(&member.thread_id).await else {
2050            return Ok(None);
2051        };
2052        self.deliver_pending_team_mailbox(team_id, member_id, member.thread_id, turn_id.clone())
2053            .await?;
2054        Ok(Some(turn_id))
2055    }
2056
2057    /// Deliver queued messages and start an idle agent, or steer its active turn.
2058    pub(crate) async fn followup_team_member(
2059        self: &Arc<Self>,
2060        caller_thread_id: &ThreadId,
2061        team_id: &str,
2062        member_id: &str,
2063        message: String,
2064    ) -> anyhow::Result<TurnId> {
2065        let _spawn_guard = self.agent_team_spawn_lock.lock().await;
2066        self.followup_team_member_locked(caller_thread_id, team_id, member_id, message)
2067            .await
2068    }
2069
2070    async fn followup_team_member_locked(
2071        self: &Arc<Self>,
2072        caller_thread_id: &ThreadId,
2073        team_id: &str,
2074        member_id: &str,
2075        message: String,
2076    ) -> anyhow::Result<TurnId> {
2077        let team = self
2078            .read_team(team_id)
2079            .await
2080            .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2081        let member = team
2082            .members
2083            .iter()
2084            .find(|member| member.id == member_id)
2085            .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2086            .clone();
2087        if member.status == TeamMemberStatus::Closed {
2088            anyhow::bail!("subagent {} is closed", member.name);
2089        }
2090        let from_member_id = team
2091            .members
2092            .iter()
2093            .find(|candidate| candidate.thread_id == *caller_thread_id)
2094            .map(|candidate| candidate.id.clone());
2095        self.teams
2096            .append_mailbox_message(
2097                team_id,
2098                from_member_id,
2099                member_id.to_string(),
2100                TeamMailboxMessageKind::NewTask,
2101                message,
2102            )
2103            .await?;
2104        if let Some(turn_id) = self.active_turn_for_thread(&member.thread_id).await {
2105            self.deliver_pending_team_mailbox(
2106                team_id,
2107                member_id,
2108                member.thread_id,
2109                turn_id.clone(),
2110            )
2111            .await?;
2112            return Ok(turn_id);
2113        }
2114
2115        self.ensure_codex_v2_team_capacity(&team, &member.id, None)
2116            .await?;
2117        let metadata = self.load_thread_metadata(&member.thread_id).await?;
2118        let inherited_context = self
2119            .team_member_turn_contexts
2120            .lock()
2121            .await
2122            .get(&member.thread_id)
2123            .cloned();
2124        let workspace = inherited_context
2125            .as_ref()
2126            .map(|context| context.workspace.clone())
2127            .or_else(|| metadata.as_ref().map(|metadata| metadata.workspace.clone()))
2128            .unwrap_or_else(|| self.workspace.display().to_string());
2129        let member_thread_id = member.thread_id;
2130        let turn_id = self
2131            .start_turn(StartTurnRequest {
2132                thread_id: member_thread_id.clone(),
2133                message: String::new(),
2134                images: Vec::new(),
2135                provider_override: member.model_provider,
2136                model_override: member.model,
2137                reasoning_override: metadata
2138                    .as_ref()
2139                    .and_then(|metadata| metadata.selection_mode.as_ref())
2140                    .and_then(ModelSelectionMode::reasoning)
2141                    .map(str::to_string),
2142                workspace,
2143                instructions: inherited_context
2144                    .as_ref()
2145                    .map(|context| context.instructions.clone())
2146                    .unwrap_or_else(crate::default_instructions),
2147                developer_context: inherited_context
2148                    .as_ref()
2149                    .and_then(|context| context.developer_context.clone()),
2150                task_ledger_required: false,
2151                service_tier_override: None,
2152            })
2153            .await?;
2154        Ok(turn_id)
2155    }
2156
2157    pub async fn set_team_member_policy_mode(
2158        &self,
2159        team_id: &str,
2160        member_id: &str,
2161        policy_mode: PolicyMode,
2162    ) -> anyhow::Result<TeamState> {
2163        self.teams
2164            .set_member_policy_mode(team_id, member_id, policy_mode)
2165            .await
2166    }
2167
2168    pub async fn interrupt_team_member(
2169        &self,
2170        team_id: &str,
2171        member_id: &str,
2172    ) -> anyhow::Result<Option<TurnId>> {
2173        let _interrupt_guard = self.agent_team_spawn_lock.lock().await;
2174        let team = self
2175            .read_team(team_id)
2176            .await
2177            .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2178        let member = team
2179            .members
2180            .iter()
2181            .find(|member| member.id == member_id)
2182            .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2183            .clone();
2184        if member.status != TeamMemberStatus::Running {
2185            return Ok(None);
2186        }
2187        let Some(turn_id) = member.current_turn_id.clone() else {
2188            return Ok(None);
2189        };
2190        if self
2191            .active_turn_for_thread(&member.thread_id)
2192            .await
2193            .as_ref()
2194            != Some(&turn_id)
2195        {
2196            return Ok(None);
2197        }
2198        self.interrupt_turn(member.thread_id.clone(), turn_id.clone())
2199            .await?;
2200        // Make queued mailbox work immediately eligible for the replacement turn while the
2201        // lifecycle lock still prevents a follow-up from starting. Delivery acknowledgements
2202        // are turn-owned, so a late acknowledgement from the aborted turn cannot steal a
2203        // reservation acquired by its replacement.
2204        self.teams
2205            .release_mailbox_reservations_for_turn(&turn_id)
2206            .await;
2207        self.teams
2208            .update_member(team_id, member_id, |member| {
2209                member.status = TeamMemberStatus::Interrupted;
2210                member.current_turn_id = None;
2211                member.terminal_error = None;
2212            })
2213            .await?;
2214        self.emit(RoderEvent::TeamMemberCompleted(TeamMemberCompleted {
2215            team_id: team_id.to_string(),
2216            member_id: member_id.to_string(),
2217            member_thread_id: member.thread_id,
2218            turn_id: Some(turn_id.clone()),
2219            status: TeamMemberStatus::Interrupted,
2220            final_message: member.final_message,
2221            error: None,
2222            timestamp: OffsetDateTime::now_utc(),
2223        }))
2224        .await;
2225        Ok(Some(turn_id))
2226    }
2227
2228    pub async fn close_team_member(
2229        &self,
2230        team_id: &str,
2231        member_id: &str,
2232    ) -> anyhow::Result<roder_api::teams::TeamMemberDescriptor> {
2233        let team = self
2234            .read_team(team_id)
2235            .await
2236            .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2237        let member = team
2238            .members
2239            .iter()
2240            .find(|member| member.id == member_id)
2241            .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2242            .clone();
2243        if member.role == roder_api::teams::TeamMemberRole::Lead {
2244            anyhow::bail!("team lead cannot be closed as a subagent");
2245        }
2246        let interrupted_turn_id = if member.status == TeamMemberStatus::Running {
2247            if let Some(turn_id) = member.current_turn_id.clone() {
2248                self.interrupt_turn(member.thread_id.clone(), turn_id.clone())
2249                    .await?;
2250                Some(turn_id)
2251            } else {
2252                None
2253            }
2254        } else {
2255            member.current_turn_id.clone()
2256        };
2257        let updated = self
2258            .teams
2259            .update_member(team_id, member_id, |member| {
2260                member.status = TeamMemberStatus::Closed;
2261                member.current_turn_id = None;
2262            })
2263            .await?;
2264        let closed = updated
2265            .members
2266            .iter()
2267            .find(|member| member.id == member_id)
2268            .cloned()
2269            .ok_or_else(|| anyhow::anyhow!("closed team member disappeared"))?;
2270        self.team_member_turn_contexts
2271            .lock()
2272            .await
2273            .remove(&closed.thread_id);
2274        self.emit(RoderEvent::TeamMemberCompleted(TeamMemberCompleted {
2275            team_id: team_id.to_string(),
2276            member_id: closed.id.clone(),
2277            member_thread_id: closed.thread_id.clone(),
2278            turn_id: interrupted_turn_id,
2279            status: TeamMemberStatus::Closed,
2280            final_message: closed.final_message.clone(),
2281            error: closed.terminal_error.clone(),
2282            timestamp: OffsetDateTime::now_utc(),
2283        }))
2284        .await;
2285        Ok(closed)
2286    }
2287
2288    pub async fn cleanup_team(&self, team_id: &str, force: bool) -> anyhow::Result<bool> {
2289        let Some(team) = self.read_team(team_id).await else {
2290            return Ok(false);
2291        };
2292        if !force
2293            && team
2294                .members
2295                .iter()
2296                .any(|member| member.status == TeamMemberStatus::Running)
2297        {
2298            anyhow::bail!("team {team_id:?} has active teammates; use forced cleanup");
2299        }
2300        if force {
2301            for member in team.members.iter().filter(|member| {
2302                member.role != roder_api::teams::TeamMemberRole::Lead
2303                    && member.status == TeamMemberStatus::Running
2304            }) {
2305                if let Some(turn_id) = member
2306                    .current_turn_id
2307                    .clone()
2308                    .or(self.active_turn_for_thread(&member.thread_id).await)
2309                {
2310                    let _ = self.interrupt_turn(member.thread_id.clone(), turn_id).await;
2311                }
2312            }
2313        }
2314        let removed = self.teams.remove(team_id).await?.is_some();
2315        if removed {
2316            let member_thread_ids = team
2317                .members
2318                .iter()
2319                .map(|member| member.thread_id.clone())
2320                .collect::<std::collections::HashSet<_>>();
2321            self.team_member_turn_contexts
2322                .lock()
2323                .await
2324                .retain(|thread_id, _| !member_thread_ids.contains(thread_id));
2325            self.emit(RoderEvent::TeamCleanupCompleted(TeamCleanupCompleted {
2326                team_id: team_id.to_string(),
2327                forced: force,
2328                timestamp: OffsetDateTime::now_utc(),
2329            }))
2330            .await;
2331        }
2332        Ok(removed)
2333    }
2334
2335    pub async fn effective_policy_mode_for_thread(&self, thread_id: &str) -> PolicyMode {
2336        if let Some(mode) = self.teams.policy_mode_for_thread(thread_id).await {
2337            return mode;
2338        }
2339        self.status().await.policy_mode
2340    }
2341
2342    async fn complete_team_member_turn_with_result(
2343        &self,
2344        thread_id: &ThreadId,
2345        turn_id: &TurnId,
2346        status: TeamMemberStatus,
2347        final_message: Option<String>,
2348        terminal_error: Option<String>,
2349    ) -> anyhow::Result<()> {
2350        let _delivery_guard = self.agent_team_spawn_lock.lock().await;
2351        let Some((team_id, member)) = self
2352            .teams
2353            .complete_member_turn(
2354                thread_id,
2355                turn_id,
2356                status,
2357                final_message.clone(),
2358                terminal_error.clone(),
2359            )
2360            .await?
2361        else {
2362            return Ok(());
2363        };
2364        if let Some(parent_thread_id) = member.parent_thread_id.as_ref()
2365            && let Some(team) = self.read_team(&team_id).await
2366            && let Some(parent) = team
2367                .members
2368                .iter()
2369                .find(|candidate| candidate.thread_id == *parent_thread_id)
2370        {
2371            let identity = member
2372                .agent_path
2373                .as_deref()
2374                .or(member.task_name.as_deref())
2375                .unwrap_or(&member.name);
2376            let mut report = format!("Agent {identity} finished with status {status:?}.");
2377            if let Some(message) = final_message.as_deref()
2378                && !message.trim().is_empty()
2379            {
2380                report.push_str("\n\nFinal result:\n");
2381                report.push_str(message);
2382            }
2383            if let Some(error) = terminal_error.as_deref()
2384                && !error.trim().is_empty()
2385            {
2386                report.push_str("\n\nTerminal error:\n");
2387                report.push_str(error);
2388            }
2389            let mailbox_appended = self
2390                .teams
2391                .append_mailbox_message(
2392                    &team_id,
2393                    Some(member.id.clone()),
2394                    parent.id.clone(),
2395                    TeamMailboxMessageKind::FinalAnswer,
2396                    report,
2397                )
2398                .await
2399                .is_ok();
2400            if mailbox_appended
2401                && let Some(parent_turn_id) = self.active_turn_for_thread(parent_thread_id).await
2402            {
2403                // The parent may finish between the active-turn lookup and enqueue. Its durable
2404                // mailbox entry remains pending for the next turn, while terminal observers must
2405                // still receive TeamMemberCompleted for this child.
2406                let _ = self
2407                    .deliver_pending_team_mailbox(
2408                        &team_id,
2409                        &parent.id,
2410                        parent_thread_id.clone(),
2411                        parent_turn_id,
2412                    )
2413                    .await;
2414            }
2415        }
2416        self.emit(RoderEvent::TeamMemberCompleted(TeamMemberCompleted {
2417            team_id,
2418            member_id: member.id,
2419            member_thread_id: member.thread_id,
2420            turn_id: Some(turn_id.clone()),
2421            status,
2422            final_message,
2423            error: terminal_error,
2424            timestamp: OffsetDateTime::now_utc(),
2425        }))
2426        .await;
2427        Ok(())
2428    }
2429
2430    /// Stops accepting new turns, requests interruption for every active turn,
2431    /// and waits for their runtime tasks to reach terminal cleanup up to
2432    /// `timeout`. Repeated calls are safe and continue draining the same set.
2433    pub async fn drain_active_turns(&self, timeout: std::time::Duration) -> RuntimeDrainOutcome {
2434        let started_at = tokio::time::Instant::now();
2435        let persistence_failures_before =
2436            self.lifecycle_persistence_failures.load(Ordering::Acquire);
2437        let turn_admission = self.turn_admission.lock().await;
2438        self.accepting_turns.store(false, Ordering::Release);
2439        let active = self
2440            .active_turns
2441            .read()
2442            .await
2443            .iter()
2444            .map(|(turn_id, handle)| (turn_id.clone(), handle.thread_id.clone()))
2445            .collect::<Vec<_>>();
2446        let draining = self
2447            .turn_drains
2448            .read()
2449            .await
2450            .iter()
2451            .map(|(turn_id, drain)| (turn_id.clone(), drain.thread_id.clone()))
2452            .collect::<Vec<_>>();
2453        drop(turn_admission);
2454
2455        let mut interrupted_turns = std::collections::BTreeMap::new();
2456        for (turn_id, thread_id) in active.iter().chain(draining.iter()) {
2457            interrupted_turns.insert(turn_id.clone(), thread_id.clone());
2458        }
2459        let interrupted_turn_ids = interrupted_turns.keys().cloned().collect::<Vec<_>>();
2460        for (turn_id, thread_id) in active {
2461            let _ = self
2462                .interrupt_turn_with_reason(thread_id, turn_id, TurnLifecycleReason::Shutdown)
2463                .await;
2464        }
2465
2466        let wait_for_empty = async {
2467            loop {
2468                let notified = self.active_turns_changed.notified();
2469                if self.active_turns.read().await.is_empty()
2470                    && self.turn_drains.read().await.is_empty()
2471                {
2472                    return;
2473                }
2474                notified.await;
2475            }
2476        };
2477        let outcome = if tokio::time::timeout(timeout, wait_for_empty).await.is_ok() {
2478            if self.lifecycle_persistence_failures.load(Ordering::Acquire)
2479                > persistence_failures_before
2480            {
2481                RuntimeDrainOutcome::PersistenceFailed {
2482                    interrupted_turn_ids,
2483                    remaining_turn_ids: Vec::new(),
2484                }
2485            } else {
2486                RuntimeDrainOutcome::Clean {
2487                    interrupted_turn_ids,
2488                }
2489            }
2490        } else {
2491            let active_remaining = self
2492                .active_turns
2493                .read()
2494                .await
2495                .iter()
2496                .map(|(turn_id, handle)| (turn_id.clone(), handle.drain.clone()))
2497                .collect::<Vec<_>>();
2498            let draining_remaining = self
2499                .turn_drains
2500                .read()
2501                .await
2502                .iter()
2503                .map(|(turn_id, drain)| (turn_id.clone(), drain.clone()))
2504                .collect::<Vec<_>>();
2505            let remaining = active_remaining
2506                .into_iter()
2507                .chain(draining_remaining)
2508                .collect::<std::collections::BTreeMap<_, _>>();
2509            let remaining_turn_ids = remaining.keys().cloned().collect::<Vec<_>>();
2510            for turn_id in &remaining_turn_ids {
2511                let handle = remaining
2512                    .get(turn_id)
2513                    .expect("remaining turn ID must have a drain handle");
2514                self.record_turn_lifecycle(
2515                    handle.thread_id.clone(),
2516                    turn_id.clone(),
2517                    TurnLifecycleState::InterruptRequested,
2518                    TurnCleanupState::TimedOut,
2519                    Some(TurnLifecycleReason::Shutdown),
2520                )
2521                .await;
2522            }
2523            if self.lifecycle_persistence_failures.load(Ordering::Acquire)
2524                > persistence_failures_before
2525            {
2526                RuntimeDrainOutcome::PersistenceFailed {
2527                    interrupted_turn_ids,
2528                    remaining_turn_ids,
2529                }
2530            } else {
2531                RuntimeDrainOutcome::DeadlineExceeded {
2532                    interrupted_turn_ids,
2533                    remaining_turn_ids,
2534                }
2535            }
2536        };
2537        self.record_lifecycle_drain_outcome(started_at, &outcome);
2538        outcome
2539    }
2540
2541    /// Enables turn submission after a prior drain attempt. This is intended
2542    /// for embedders that keep a runtime alive after a bounded drain timeout.
2543    pub fn resume_accepting_turns(&self) {
2544        self.accepting_turns.store(true, Ordering::Release);
2545    }
2546
2547    pub async fn turn_lifecycle_snapshot(
2548        &self,
2549        thread_id: &ThreadId,
2550    ) -> anyhow::Result<TurnLifecycleSnapshot> {
2551        let Some(store) = &self.thread_store else {
2552            return Ok(TurnLifecycleSnapshot::default());
2553        };
2554        let extension_states = store.load_extension_states(thread_id).await?;
2555        Ok(turn_lifecycle_snapshot(&extension_states))
2556    }
2557
2558    pub async fn load_thread(
2559        &self,
2560        thread_id: &ThreadId,
2561    ) -> anyhow::Result<Option<ThreadSnapshot>> {
2562        let loaded = if let Some(store) = &self.thread_store {
2563            store.load_thread(thread_id).await?
2564        } else {
2565            None
2566        };
2567        if let Some(snapshot) = loaded.as_ref() {
2568            let live_turn_ids = self
2569                .active_turns
2570                .read()
2571                .await
2572                .keys()
2573                .cloned()
2574                .chain(self.turn_drains.read().await.keys().cloned())
2575                .collect::<std::collections::HashSet<_>>();
2576            let lifecycle = turn_lifecycle_snapshot(&snapshot.extension_states);
2577            for record in lifecycle.records.into_iter().filter(|record| {
2578                record.state.requires_recovery() && !live_turn_ids.contains(&record.turn_id)
2579            }) {
2580                self.lifecycle_restart_reconciliations
2581                    .fetch_add(1, Ordering::AcqRel);
2582                self.record_turn_lifecycle(
2583                    record.thread_id,
2584                    record.turn_id,
2585                    TurnLifecycleState::RecoveryNeeded,
2586                    TurnCleanupState::Unknown,
2587                    Some(TurnLifecycleReason::RuntimeRestart),
2588                )
2589                .await;
2590            }
2591            self.emit(RoderEvent::ThreadLoaded(ThreadLoaded {
2592                thread_id: thread_id.clone(),
2593                timestamp: OffsetDateTime::now_utc(),
2594            }))
2595            .await;
2596        }
2597        Ok(loaded)
2598    }
2599
2600    pub async fn workspace_for_thread(&self, thread_id: &ThreadId) -> anyhow::Result<String> {
2601        if let Some(store) = &self.thread_store {
2602            let snapshot = store
2603                .load_thread(thread_id)
2604                .await?
2605                .ok_or_else(|| anyhow::anyhow!("thread not found: {thread_id}"));
2606            match snapshot {
2607                Ok(snapshot) => {
2608                    if let Some(metadata) = snapshot.metadata {
2609                        // Fork-backed threads fail closed before any write
2610                        // when their workspace was removed out-of-band.
2611                        if let Some(fork) = &metadata.workspace_fork
2612                            && fork.status == roder_api::forks::ForkStatus::Active
2613                            && !std::path::Path::new(&metadata.workspace).is_dir()
2614                        {
2615                            anyhow::bail!(
2616                                "workspace fork {} is missing its workspace at {}; restore it or \
2617                                 remove the fork before running turns in this thread",
2618                                fork.id,
2619                                metadata.workspace
2620                            );
2621                        }
2622                        return Ok(metadata.workspace);
2623                    }
2624                    eprintln!(
2625                        "thread metadata missing while resolving workspace for {thread_id}; falling back to runtime workspace"
2626                    );
2627                }
2628                Err(err) => {
2629                    eprintln!(
2630                        "thread missing while resolving workspace for {thread_id}: {err}; falling back to runtime workspace"
2631                    );
2632                }
2633            }
2634        }
2635        Ok(self.workspace.display().to_string())
2636    }
2637
2638    pub(crate) async fn selection_mode_for_thread(
2639        &self,
2640        thread_id: &ThreadId,
2641    ) -> anyhow::Result<Option<ModelSelectionMode>> {
2642        let Some(store) = &self.thread_store else {
2643            return Ok(None);
2644        };
2645        Ok(store
2646            .load_thread(thread_id)
2647            .await?
2648            .and_then(|snapshot| snapshot.metadata)
2649            .and_then(|metadata| {
2650                metadata
2651                    .selection_mode
2652                    .or_else(|| match (metadata.provider, metadata.model) {
2653                        (Some(provider), Some(model)) => {
2654                            Some(ModelSelectionMode::manual(provider, model, None))
2655                        }
2656                        _ => None,
2657                    })
2658            }))
2659    }
2660
2661    /// Concrete provider/model for configured-role subagents (`task`,
2662    /// `agent_swarm`). Prefers the live parent turn selection so children stay
2663    /// on SuperGrok/Grok (or whatever the lead is using) instead of process
2664    /// startup defaults such as openai/gpt-5.5.
2665    pub(crate) async fn parent_model_selection_for_subagents(
2666        &self,
2667        thread_id: &ThreadId,
2668        turn_id: &TurnId,
2669    ) -> Option<roder_api::inference::ModelSelection> {
2670        if let Some(selection) = self
2671            .active_turn_selections
2672            .read()
2673            .await
2674            .get(turn_id)
2675            .cloned()
2676        {
2677            return Some(selection.concrete_selection());
2678        }
2679        if let Ok(Some(selection)) = self.selection_mode_for_thread(thread_id).await {
2680            return Some(selection.concrete_selection());
2681        }
2682        let cfg = self.status().await;
2683        if cfg.default_provider.trim().is_empty() || cfg.default_model.trim().is_empty() {
2684            return None;
2685        }
2686        Some(roder_api::inference::ModelSelection {
2687            provider: cfg.default_provider,
2688            model: cfg.default_model,
2689        })
2690    }
2691
2692    /// Per-thread tool allowlist, developer instructions, and external tools persisted at thread creation.
2693    pub(crate) async fn thread_turn_overrides(
2694        &self,
2695        thread_id: &ThreadId,
2696    ) -> anyhow::Result<ThreadTurnOverrides> {
2697        let Some(store) = &self.thread_store else {
2698            return Ok(ThreadTurnOverrides::default());
2699        };
2700        Ok(store
2701            .load_thread_metadata(thread_id)
2702            .await?
2703            .map(|metadata| ThreadTurnOverrides {
2704                tool_allowlist: metadata.tool_allowlist,
2705                developer_instructions: metadata.developer_instructions,
2706                external_tools: metadata.external_tools,
2707            })
2708            .unwrap_or_default())
2709    }
2710
2711    pub async fn set_thread_selection_mode(
2712        &self,
2713        thread_id: &ThreadId,
2714        selection_mode: ModelSelectionMode,
2715    ) -> anyhow::Result<()> {
2716        let Some(store) = &self.thread_store else {
2717            return Ok(());
2718        };
2719        let Some(snapshot) = store.load_thread(thread_id).await? else {
2720            anyhow::bail!("thread not found: {thread_id}");
2721        };
2722        let Some(mut metadata) = snapshot.metadata else {
2723            return Ok(());
2724        };
2725        let concrete = selection_mode.concrete_selection();
2726        metadata.provider = Some(concrete.provider);
2727        metadata.model = Some(concrete.model);
2728        metadata.selection_mode = Some(selection_mode);
2729        metadata.updated_at = OffsetDateTime::now_utc();
2730        store.update_thread_metadata(metadata).await?;
2731        Ok(())
2732    }
2733
2734    async fn runner_session_for_thread(
2735        &self,
2736        thread_id: &ThreadId,
2737    ) -> anyhow::Result<Option<(RunnerDestination, Arc<dyn RemoteRunnerSession>)>> {
2738        let metadata = if let Some(store) = &self.thread_store {
2739            store.load_thread_metadata(thread_id).await?
2740        } else {
2741            None
2742        };
2743        // An explicit per-thread binding wins over the runtime-level destination.
2744        let destination = metadata
2745            .as_ref()
2746            .and_then(|metadata| metadata.runner_binding.as_ref())
2747            .map(|binding| binding.destination.clone())
2748            .or(self.config.read().await.remote_runner_destination.clone());
2749        let Some(destination) = destination else {
2750            self.evict_runner_session(thread_id).await;
2751            return Ok(None);
2752        };
2753        let provider = self
2754            .registry
2755            .remote_runner_providers
2756            .iter()
2757            .find(|provider| provider.id() == destination.provider_id)
2758            .cloned()
2759            .ok_or_else(|| {
2760                anyhow::anyhow!(
2761                    "remote runner provider {:?} is not installed",
2762                    destination.provider_id
2763                )
2764            })?;
2765        let persisted_state = metadata.and_then(|metadata| metadata.runner_state);
2766        let slot = {
2767            let mut sessions = self.runner_sessions.lock().await;
2768            match sessions.get(thread_id) {
2769                Some(slot) if slot.matches(&destination) => slot.clone(),
2770                _ => {
2771                    let slot = Arc::new(RunnerSessionSlot::empty(&destination));
2772                    sessions.insert(thread_id.clone(), slot.clone());
2773                    slot
2774                }
2775            }
2776        };
2777
2778        loop {
2779            let initialized = slot.initialized.notified();
2780            let mut state = slot.state.lock().await;
2781            match &*state {
2782                RunnerSessionSlotState::Ready(cached) => {
2783                    return Ok(Some((cached.destination.clone(), cached.session.clone())));
2784                }
2785                RunnerSessionSlotState::Failed(error) => {
2786                    return Err(anyhow::anyhow!(error.to_string()));
2787                }
2788                RunnerSessionSlotState::Initializing => {
2789                    drop(state);
2790                    initialized.await;
2791                }
2792                RunnerSessionSlotState::Empty => {
2793                    *state = RunnerSessionSlotState::Initializing;
2794                    drop(state);
2795                    self.spawn_runner_session_initialization(
2796                        thread_id.clone(),
2797                        destination.clone(),
2798                        provider.clone(),
2799                        persisted_state.clone(),
2800                        slot.clone(),
2801                    );
2802                }
2803            }
2804        }
2805    }
2806
2807    fn spawn_runner_session_initialization(
2808        &self,
2809        thread_id: ThreadId,
2810        destination: RunnerDestination,
2811        provider: Arc<dyn RemoteRunnerProvider>,
2812        persisted_state: Option<RunnerSessionState>,
2813        slot: Arc<RunnerSessionSlot>,
2814    ) {
2815        let thread_store = self.thread_store.clone();
2816        let runner_sessions = self.runner_sessions.clone();
2817        // The task intentionally outlives the turn that first requested the
2818        // workspace. Interrupting that turn must not strand the slot in
2819        // `Initializing` and deadlock every later tool or lifecycle call.
2820        drop(tokio::spawn(async move {
2821            let initialized = async {
2822                let session = if let Some(state) = persisted_state
2823                    && state.provider_id == destination.provider_id
2824                    && state.destination_id == destination.id
2825                {
2826                    match provider.resume_session(state).await {
2827                        Ok(session) => session,
2828                        Err(_) => provider.create_session(destination.clone()).await?,
2829                    }
2830                } else {
2831                    provider.create_session(destination.clone()).await?
2832                };
2833                let resolved = (destination.clone(), session.clone());
2834                // Persist before releasing the singleflight or dispatching
2835                // the first tool. A later process can rejoin even if the turn
2836                // that requested initialization has already been interrupted.
2837                Self::persist_runner_state_in_store(
2838                    thread_store.as_ref(),
2839                    &thread_id,
2840                    Some(&resolved),
2841                )
2842                .await?;
2843                Ok::<_, anyhow::Error>(CachedRunnerSession {
2844                    destination,
2845                    session,
2846                })
2847            }
2848            .await;
2849
2850            match initialized {
2851                Ok(cached) => {
2852                    *slot.state.lock().await = RunnerSessionSlotState::Ready(cached);
2853                    slot.initialized.notify_waiters();
2854                }
2855                Err(error) => {
2856                    let message: Arc<str> = Arc::from(format!("{error:#}"));
2857                    *slot.state.lock().await = RunnerSessionSlotState::Failed(message);
2858                    slot.initialized.notify_waiters();
2859                    let mut sessions = runner_sessions.lock().await;
2860                    let is_current = sessions
2861                        .get(&thread_id)
2862                        .is_some_and(|current| Arc::ptr_eq(current, &slot));
2863                    if is_current {
2864                        sessions.remove(&thread_id);
2865                    }
2866                }
2867            }
2868        }));
2869    }
2870
2871    async fn evict_runner_session(&self, thread_id: &ThreadId) {
2872        self.runner_sessions.lock().await.remove(thread_id);
2873    }
2874
2875    async fn cache_runner_session(
2876        &self,
2877        thread_id: &ThreadId,
2878        destination: RunnerDestination,
2879        session: Arc<dyn RemoteRunnerSession>,
2880    ) {
2881        let cached = CachedRunnerSession {
2882            destination,
2883            session,
2884        };
2885        self.runner_sessions.lock().await.insert(
2886            thread_id.clone(),
2887            Arc::new(RunnerSessionSlot::ready(cached)),
2888        );
2889    }
2890
2891    /// Read a thread's explicit runner binding without creating or resuming a
2892    /// session. Tool routing uses this before local previews so runner-backed
2893    /// and fail-closed hosted calls never inspect the host workspace.
2894    pub(crate) async fn runner_binding_for_thread(
2895        &self,
2896        thread_id: &ThreadId,
2897    ) -> anyhow::Result<Option<ThreadRunnerBinding>> {
2898        let Some(store) = &self.thread_store else {
2899            return Ok(None);
2900        };
2901        Ok(store
2902            .load_thread_metadata(thread_id)
2903            .await?
2904            .and_then(|metadata| metadata.runner_binding))
2905    }
2906
2907    /// Resolve a previously-read binding into a live remote workspace. The
2908    /// session remains lazy: callers invoke this only after policy approval.
2909    pub(crate) async fn remote_workspace_for_binding(
2910        &self,
2911        thread_id: &ThreadId,
2912        binding: ThreadRunnerBinding,
2913    ) -> anyhow::Result<Arc<RemoteWorkspace>> {
2914        let session = self
2915            .runner_session_for_thread(thread_id)
2916            .await?
2917            .map(|(_, session)| session)
2918            .ok_or_else(|| {
2919                anyhow::anyhow!("runner-bound thread {thread_id} has no runner session")
2920            })?;
2921        Ok(Arc::new(RemoteWorkspace {
2922            session,
2923            root: binding.workspace,
2924            read_roots: binding.read_roots,
2925        }))
2926    }
2927
2928    pub(crate) async fn runner_tool_execution_lock(
2929        &self,
2930        session: &dyn RemoteRunnerSession,
2931    ) -> Arc<RunnerToolExecutionMutex> {
2932        let state = session.state();
2933        let key = (state.provider_id, state.session_id);
2934        let mut locks = self.runner_tool_execution_locks.lock().await;
2935        if let Some(lock) = locks.get(&key).and_then(Weak::upgrade) {
2936            return lock;
2937        }
2938
2939        locks.retain(|_, lock| lock.strong_count() > 0);
2940        let lock = Arc::new(Mutex::new(()));
2941        locks.insert(key, Arc::downgrade(&lock));
2942        lock
2943    }
2944
2945    async fn persist_runner_state(
2946        &self,
2947        thread_id: &ThreadId,
2948        runner: Option<&(RunnerDestination, Arc<dyn RemoteRunnerSession>)>,
2949    ) -> anyhow::Result<()> {
2950        Self::persist_runner_state_in_store(self.thread_store.as_ref(), thread_id, runner).await
2951    }
2952
2953    async fn persist_runner_state_in_store(
2954        store: Option<&Arc<dyn ThreadStore>>,
2955        thread_id: &ThreadId,
2956        runner: Option<&(RunnerDestination, Arc<dyn RemoteRunnerSession>)>,
2957    ) -> anyhow::Result<()> {
2958        let Some((destination, session)) = runner else {
2959            return Ok(());
2960        };
2961        let Some(store) = store else {
2962            return Ok(());
2963        };
2964        let Some(snapshot) = store.load_thread(thread_id).await? else {
2965            return Ok(());
2966        };
2967        let Some(mut metadata) = snapshot.metadata else {
2968            return Ok(());
2969        };
2970        metadata.runner_destination = Some(destination.clone());
2971        metadata.runner_state = Some(session.state());
2972        metadata.updated_at = OffsetDateTime::now_utc();
2973        store.update_thread_metadata(metadata).await?;
2974        Ok(())
2975    }
2976
2977    fn remote_runner_provider_by_id(
2978        &self,
2979        provider_id: &str,
2980    ) -> Option<Arc<dyn RemoteRunnerProvider>> {
2981        self.registry
2982            .remote_runner_providers
2983            .iter()
2984            .find(|provider| provider.id() == provider_id)
2985            .cloned()
2986    }
2987
2988    /// Resolve the live runner session for a thread along with its provider,
2989    /// failing clearly when the thread is not runner-bound.
2990    async fn thread_runner_session(
2991        &self,
2992        thread_id: &ThreadId,
2993    ) -> anyhow::Result<(
2994        RunnerDestination,
2995        Arc<dyn RemoteRunnerProvider>,
2996        Arc<dyn RemoteRunnerSession>,
2997    )> {
2998        let Some((destination, session)) = self.runner_session_for_thread(thread_id).await? else {
2999            anyhow::bail!("thread {thread_id} is not bound to a remote runner");
3000        };
3001        let provider = self
3002            .remote_runner_provider_by_id(&destination.provider_id)
3003            .ok_or_else(|| {
3004                anyhow::anyhow!(
3005                    "remote runner provider {:?} is not installed",
3006                    destination.provider_id
3007                )
3008            })?;
3009        Ok((destination, provider, session))
3010    }
3011
3012    /// Pause a runner-bound thread's session toward standby and persist the
3013    /// post-pause state. Errors if the provider is not pausable.
3014    pub async fn pause_thread_runner(
3015        &self,
3016        thread_id: &ThreadId,
3017    ) -> anyhow::Result<RunnerSessionState> {
3018        let (destination, provider, session) = self.thread_runner_session(thread_id).await?;
3019        anyhow::ensure!(
3020            provider.capabilities().pausable,
3021            "remote runner provider {:?} does not support pausing",
3022            destination.provider_id
3023        );
3024        let state = session.pause().await?;
3025        self.persist_runner_state(thread_id, Some(&(destination, session)))
3026            .await?;
3027        Ok(state)
3028    }
3029
3030    /// Resume (wake) a runner-bound thread's paused session and persist state.
3031    pub async fn resume_thread_runner(
3032        &self,
3033        thread_id: &ThreadId,
3034    ) -> anyhow::Result<RunnerSessionState> {
3035        let (destination, _provider, session) = self.thread_runner_session(thread_id).await?;
3036        let state = session.resume().await?;
3037        self.persist_runner_state(thread_id, Some(&(destination, session)))
3038            .await?;
3039        Ok(state)
3040    }
3041
3042    /// Detach a runner-bound thread's session: persist the durable, rejoinable
3043    /// state and leave the remote sandbox alive. Errors if not detachable.
3044    pub async fn detach_thread_runner(
3045        &self,
3046        thread_id: &ThreadId,
3047    ) -> anyhow::Result<RunnerSessionState> {
3048        let (destination, provider, session) = self.thread_runner_session(thread_id).await?;
3049        anyhow::ensure!(
3050            provider.capabilities().detachable,
3051            "remote runner provider {:?} does not support detaching",
3052            destination.provider_id
3053        );
3054        let state = session.detach().await?;
3055        // Persist the detached state explicitly so a later turn or process
3056        // rejoins the same sandbox instead of provisioning a new one.
3057        if let Some(store) = &self.thread_store
3058            && let Some(snapshot) = store.load_thread(thread_id).await?
3059            && let Some(mut metadata) = snapshot.metadata
3060        {
3061            metadata.runner_destination = Some(destination);
3062            metadata.runner_state = Some(state.clone());
3063            metadata.updated_at = OffsetDateTime::now_utc();
3064            store.update_thread_metadata(metadata).await?;
3065        }
3066        self.evict_runner_session(thread_id).await;
3067        Ok(state)
3068    }
3069
3070    /// Rejoin a previously created sandbox from a thread's persisted runner
3071    /// state without provisioning a new one. An optional `sandbox` overrides the
3072    /// persisted sandbox name (recovery by name). Persists refreshed state.
3073    pub async fn rejoin_thread_runner(
3074        &self,
3075        thread_id: &ThreadId,
3076        sandbox: Option<String>,
3077    ) -> anyhow::Result<RunnerSessionState> {
3078        let store = self
3079            .thread_store
3080            .as_ref()
3081            .ok_or_else(|| anyhow::anyhow!("thread store is required to rejoin a runner"))?;
3082        let metadata = store
3083            .load_thread_metadata(thread_id)
3084            .await?
3085            .ok_or_else(|| anyhow::anyhow!("thread {thread_id} has no metadata"))?;
3086        let destination = metadata
3087            .runner_binding
3088            .as_ref()
3089            .map(|binding| binding.destination.clone())
3090            .or_else(|| metadata.runner_destination.clone())
3091            .ok_or_else(|| anyhow::anyhow!("thread {thread_id} is not bound to a remote runner"))?;
3092        let mut state = metadata
3093            .runner_state
3094            .clone()
3095            .ok_or_else(|| anyhow::anyhow!("thread {thread_id} has no persisted runner state"))?;
3096        if let Some(sandbox) = sandbox
3097            && let Some(object) = state.metadata.as_object_mut()
3098        {
3099            object.insert("sandbox_name".to_string(), serde_json::Value::from(sandbox));
3100        }
3101        let provider = self
3102            .remote_runner_provider_by_id(&destination.provider_id)
3103            .ok_or_else(|| {
3104                anyhow::anyhow!(
3105                    "remote runner provider {:?} is not installed",
3106                    destination.provider_id
3107                )
3108            })?;
3109        let session = provider.rejoin_session(state).await?;
3110        self.persist_runner_state(thread_id, Some(&(destination.clone(), session.clone())))
3111            .await?;
3112        self.cache_runner_session(thread_id, destination, session.clone())
3113            .await;
3114        Ok(session.state())
3115    }
3116
3117    pub(crate) async fn record_thread_usage_metadata(
3118        &self,
3119        thread_id: &ThreadId,
3120        usage: &TokenUsage,
3121    ) -> anyhow::Result<()> {
3122        if usage.is_empty() {
3123            return Ok(());
3124        }
3125        let Some(store) = &self.thread_store else {
3126            return Ok(());
3127        };
3128        let Some(snapshot) = store.load_thread(thread_id).await? else {
3129            return Ok(());
3130        };
3131        let Some(mut metadata) = snapshot.metadata else {
3132            return Ok(());
3133        };
3134        metadata
3135            .usage
3136            .get_or_insert_with(ThreadUsageMetadata::default)
3137            .add_token_usage(usage);
3138        metadata.updated_at = OffsetDateTime::now_utc();
3139        store.update_thread_metadata(metadata).await?;
3140        Ok(())
3141    }
3142
3143    pub(crate) async fn has_active_turn_for_thread(&self, thread_id: &ThreadId) -> bool {
3144        self.active_turns
3145            .read()
3146            .await
3147            .values()
3148            .any(|handle| &handle.thread_id == thread_id)
3149    }
3150
3151    async fn ensure_codex_v2_team_capacity(
3152        &self,
3153        team: &TeamState,
3154        resuming_member_id: &str,
3155        candidate_model: Option<&str>,
3156    ) -> anyhow::Result<()> {
3157        if !is_codex_v2_team(team)
3158            && !candidate_model
3159                .is_some_and(|model| model_supports_reasoning_effort(model, REASONING_ULTRA))
3160        {
3161            return Ok(());
3162        }
3163        let active_thread_ids = self
3164            .active_turns
3165            .read()
3166            .await
3167            .values()
3168            .map(|handle| handle.thread_id.clone())
3169            .collect::<std::collections::HashSet<_>>();
3170        let resident_threads = 1 + team
3171            .members
3172            .iter()
3173            .filter(|member| {
3174                member.role != roder_api::teams::TeamMemberRole::Lead
3175                    && member.id != resuming_member_id
3176                    && active_thread_ids.contains(&member.thread_id)
3177            })
3178            .count();
3179        anyhow::ensure!(
3180            resident_threads < codex_v2::CODEX_V2_MAX_RESIDENT_TEAM_THREADS,
3181            "agent thread limit reached for this Codex V2 team: maximum {} resident threads (the lead plus 3 running subagents)",
3182            codex_v2::CODEX_V2_MAX_RESIDENT_TEAM_THREADS
3183        );
3184        Ok(())
3185    }
3186
3187    /// Number of currently running turns across all threads. Hosted runtime
3188    /// pools use this to avoid evicting tenants with active work.
3189    pub async fn active_turn_count(&self) -> usize {
3190        self.active_turns.read().await.len()
3191    }
3192
3193    pub async fn active_turn_for_thread(&self, thread_id: &ThreadId) -> Option<TurnId> {
3194        self.active_turns
3195            .read()
3196            .await
3197            .iter()
3198            .find_map(|(turn_id, handle)| (&handle.thread_id == thread_id).then(|| turn_id.clone()))
3199    }
3200
3201    pub async fn thread_activity(&self, thread_id: &ThreadId) -> ThreadActivity {
3202        let active_turn_id = self.active_turn_for_thread(thread_id).await;
3203        let draining_turn_id = if active_turn_id.is_none() {
3204            self.turn_drains
3205                .read()
3206                .await
3207                .iter()
3208                .find_map(|(turn_id, drain)| {
3209                    (&drain.thread_id == thread_id).then(|| turn_id.clone())
3210                })
3211        } else {
3212            None
3213        };
3214        let Some(active_turn_id) = active_turn_id.or(draining_turn_id) else {
3215            return ThreadActivity::default();
3216        };
3217
3218        let mut active_flags = Vec::new();
3219        {
3220            let pending_approvals = self.pending_tool_approvals.lock().await;
3221            if pending_approvals
3222                .values()
3223                .any(|pending| &pending.thread_id == thread_id && pending.turn_id == active_turn_id)
3224            {
3225                active_flags.push("approvalRequired".to_string());
3226            }
3227        }
3228        {
3229            let pending_inputs = self.pending_user_inputs.lock().await;
3230            if pending_inputs
3231                .values()
3232                .any(|pending| &pending.thread_id == thread_id && pending.turn_id == active_turn_id)
3233            {
3234                active_flags.push("userInputRequired".to_string());
3235            }
3236        }
3237        {
3238            let pending_external = self.pending_external_tool_calls.lock().await;
3239            if pending_external
3240                .values()
3241                .any(|pending| &pending.thread_id == thread_id && pending.turn_id == active_turn_id)
3242            {
3243                active_flags.push("externalToolPending".to_string());
3244            }
3245        }
3246        let interrupting = self
3247            .active_turns
3248            .read()
3249            .await
3250            .get(&active_turn_id)
3251            .is_some_and(|handle| handle.drain.interrupt_requested.load(Ordering::Acquire))
3252            || self
3253                .turn_drains
3254                .read()
3255                .await
3256                .get(&active_turn_id)
3257                .is_some_and(|drain| drain.interrupt_requested.load(Ordering::Acquire));
3258        if interrupting {
3259            active_flags.push("interrupting".to_string());
3260        }
3261        if self.pending_plan_exit().await.is_some_and(|pending| {
3262            &pending.thread_id == thread_id && pending.turn_id == active_turn_id
3263        }) {
3264            active_flags.push("planExitRequired".to_string());
3265        }
3266
3267        ThreadActivity {
3268            active_turn_id: Some(active_turn_id),
3269            active_flags,
3270        }
3271    }
3272
3273    pub async fn interrupt_turn(&self, thread_id: ThreadId, turn_id: TurnId) -> anyhow::Result<()> {
3274        self.interrupt_turn_with_reason(thread_id, turn_id, TurnLifecycleReason::UserInterrupt)
3275            .await
3276    }
3277
3278    async fn interrupt_turn_with_reason(
3279        &self,
3280        thread_id: ThreadId,
3281        turn_id: TurnId,
3282        reason: TurnLifecycleReason,
3283    ) -> anyhow::Result<()> {
3284        let handle = self.active_turns.write().await.remove(&turn_id);
3285        if let Some(handle) = handle {
3286            if handle
3287                .drain
3288                .interrupt_requested
3289                .swap(true, Ordering::AcqRel)
3290            {
3291                return Ok(());
3292            }
3293            *handle.drain.interrupt_reason.lock().await = Some(reason);
3294            self.turn_drains
3295                .write()
3296                .await
3297                .insert(turn_id.clone(), handle.drain.clone());
3298            let ownership = self.provider_cleanup_ownership(&turn_id);
3299            self.record_turn_lifecycle_with_ownership(
3300                thread_id.clone(),
3301                turn_id.clone(),
3302                TurnLifecycleState::InterruptRequested,
3303                TurnCleanupState::Requested,
3304                Some(reason),
3305                ownership,
3306            )
3307            .await;
3308            handle.abort.abort();
3309            if reason == TurnLifecycleReason::UserInterrupt {
3310                self.goals
3311                    .finish_turn(
3312                        &thread_id,
3313                        &turn_id,
3314                        Some(roder_api::goals::ThreadGoalStatus::Paused),
3315                    )
3316                    .await?;
3317            }
3318            self.active_turns_changed.notify_waiters();
3319        }
3320        self.active_turn_selections.write().await.remove(&turn_id);
3321        self.cancel_pending_external_tool_calls_for_turn(&turn_id)
3322            .await;
3323        Ok(())
3324    }
3325
3326    pub async fn steer_turn(
3327        &self,
3328        thread_id: ThreadId,
3329        turn_id: TurnId,
3330        message: String,
3331        images: Vec<InputImage>,
3332    ) -> anyhow::Result<()> {
3333        self.enqueue_turn_steer(thread_id, turn_id, message, images, None)
3334            .await
3335    }
3336
3337    pub(crate) async fn has_pending_turn_steers(&self, turn_id: &TurnId) -> bool {
3338        let active = self.active_turns.read().await.get(turn_id).cloned();
3339        let Some(active) = active else {
3340            return false;
3341        };
3342        let has_pending = !active.steers.lock().await.is_empty();
3343        has_pending
3344    }
3345
3346    async fn steer_turn_with_mailbox_ack(
3347        &self,
3348        thread_id: ThreadId,
3349        turn_id: TurnId,
3350        message: String,
3351        message_ids: Vec<String>,
3352        team_id: TeamId,
3353    ) -> anyhow::Result<()> {
3354        self.enqueue_turn_steer(
3355            thread_id,
3356            turn_id,
3357            message,
3358            Vec::new(),
3359            Some(MailboxDeliveryAck {
3360                team_id,
3361                message_ids,
3362            }),
3363        )
3364        .await
3365    }
3366
3367    async fn deliver_pending_team_mailbox(
3368        &self,
3369        team_id: &str,
3370        member_id: &str,
3371        member_thread_id: ThreadId,
3372        turn_id: TurnId,
3373    ) -> anyhow::Result<()> {
3374        let pending = self
3375            .teams
3376            .reserve_pending_mailbox_messages(team_id, member_id, &turn_id)
3377            .await?;
3378        if pending.is_empty() {
3379            return Ok(());
3380        }
3381        let message_ids = pending
3382            .iter()
3383            .map(|message| message.id.clone())
3384            .collect::<Vec<_>>();
3385        let team = self
3386            .read_team(team_id)
3387            .await
3388            .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
3389        if let Err(error) = self
3390            .steer_turn_with_mailbox_ack(
3391                member_thread_id,
3392                turn_id.clone(),
3393                format_mailbox_messages(&team, &pending),
3394                message_ids.clone(),
3395                team_id.to_string(),
3396            )
3397            .await
3398        {
3399            self.teams
3400                .release_mailbox_reservations(&turn_id, &message_ids)
3401                .await;
3402            return Err(error);
3403        }
3404        Ok(())
3405    }
3406
3407    async fn enqueue_turn_steer(
3408        &self,
3409        thread_id: ThreadId,
3410        turn_id: TurnId,
3411        message: String,
3412        images: Vec<InputImage>,
3413        mailbox_ack: Option<MailboxDeliveryAck>,
3414    ) -> anyhow::Result<()> {
3415        let message = message.trim().to_string();
3416        if message.is_empty() && images.is_empty() {
3417            return Ok(());
3418        }
3419
3420        let Some(active) = self.active_turns.read().await.get(&turn_id).cloned() else {
3421            anyhow::bail!("no active turn to steer");
3422        };
3423        anyhow::ensure!(
3424            active.thread_id == thread_id,
3425            "turn does not belong to thread"
3426        );
3427        {
3428            let mut steers = active.steers.lock().await;
3429            steers.push(QueuedTurnSteer {
3430                message: UserMessage::with_images(message.clone(), images),
3431                mailbox_ack,
3432            });
3433            active
3434                .steer_changed
3435                .send_modify(|generation| *generation = generation.wrapping_add(1));
3436        }
3437        self.emit(RoderEvent::TurnSteered(TurnSteered {
3438            thread_id,
3439            turn_id,
3440            message,
3441            timestamp: OffsetDateTime::now_utc(),
3442        }))
3443        .await;
3444        Ok(())
3445    }
3446
3447    pub async fn tool_specs(&self) -> Vec<roder_api::tools::ToolSpec> {
3448        let cfg = self.config.read().await;
3449        let model_profile =
3450            model_profile_for_provider_model(&cfg, &cfg.default_provider, &cfg.default_model);
3451        self.filtered_tool_specs(
3452            &cfg,
3453            &cfg.default_model,
3454            &cfg.default_provider,
3455            model_profile.as_ref(),
3456            &[],
3457            &[],
3458        )
3459    }
3460
3461    pub fn subagent_definitions(&self) -> Vec<SubagentDefinition> {
3462        self.registry
3463            .subagent_dispatchers
3464            .iter()
3465            .flat_map(|dispatcher| dispatcher.definitions())
3466            .collect()
3467    }
3468
3469    async fn run_turn(
3470        self: &Arc<Self>,
3471        req: StartTurnRequest,
3472        turn_id: TurnId,
3473        initial_mailbox_ack: Option<MailboxDeliveryAck>,
3474        mut steering: tokio::sync::watch::Receiver<u64>,
3475        automatic_goal_turn: bool,
3476    ) -> anyhow::Result<TurnRunOutcome> {
3477        self.ensure_execution_authority()?;
3478        self.goals
3479            .begin_turn(&req.thread_id, &turn_id, automatic_goal_turn)
3480            .await?;
3481        let turn_started_at = OffsetDateTime::now_utc();
3482        self.emit(RoderEvent::TurnStarted(TurnStarted {
3483            thread_id: req.thread_id.clone(),
3484            turn_id: turn_id.clone(),
3485            runtime_profile: self.config.read().await.runtime_profile,
3486            timestamp: turn_started_at,
3487        }))
3488        .await;
3489        self.persist_turn_item(
3490            &req.thread_id,
3491            &turn_id,
3492            &TranscriptItem::UserMessage(UserMessage::with_images(
3493                req.message.clone(),
3494                req.images.clone(),
3495            )),
3496        )
3497        .await?;
3498        crate::hooks::run_lifecycle(
3499            self,
3500            &req.thread_id,
3501            &turn_id,
3502            Some(&req.workspace),
3503            "UserPromptSubmit",
3504            None,
3505            serde_json::json!({"prompt": req.message}),
3506        )
3507        .await;
3508        if let Some(ack) = initial_mailbox_ack {
3509            self.teams
3510                .mark_mailbox_messages_delivered(&ack.team_id, &turn_id, &ack.message_ids)
3511                .await?;
3512        }
3513
3514        let mut cfg = self.config.read().await.clone();
3515        let runtime_profile = cfg.runtime_profile;
3516        let turn_deadline = turn_deadline_for_config(&cfg);
3517        let deadline_finalization_reserve =
3518            crate::deadline_policy::finalization_reserve_seconds(cfg.turn_deadline_seconds);
3519        let selection_mode = self.selection_mode_for_thread(&req.thread_id).await?;
3520        let concrete_selection = selection_mode
3521            .as_ref()
3522            .map(ModelSelectionMode::concrete_selection);
3523        let default_provider = req
3524            .provider_override
3525            .clone()
3526            .or_else(|| {
3527                concrete_selection
3528                    .as_ref()
3529                    .map(|selection| selection.provider.clone())
3530            })
3531            .unwrap_or(cfg.default_provider.clone());
3532        let default_model = req
3533            .model_override
3534            .clone()
3535            .or_else(|| {
3536                concrete_selection
3537                    .as_ref()
3538                    .map(|selection| selection.model.clone())
3539            })
3540            .unwrap_or(cfg.default_model.clone());
3541        if let Some(reasoning) = req.reasoning_override.as_deref().or_else(|| {
3542            selection_mode
3543                .as_ref()
3544                .and_then(ModelSelectionMode::reasoning)
3545        }) {
3546            validate_reasoning_effort(&default_model, reasoning)?;
3547            cfg.reasoning = Some(reasoning.to_string());
3548        }
3549        let turn_has_concrete_model_override =
3550            req.provider_override.is_some() || req.model_override.is_some();
3551        let (turn_inference_router, turn_inference_router_profile) = match &selection_mode {
3552            Some(ModelSelectionMode::Auto {
3553                router_id, profile, ..
3554            }) if !turn_has_concrete_model_override => (
3555                RuntimeInferenceRouterConfig {
3556                    enabled: true,
3557                    router_id: Some(router_id.clone()),
3558                },
3559                profile.clone(),
3560            ),
3561            _ => (RuntimeInferenceRouterConfig::disabled(), None),
3562        };
3563        let mut provider = default_provider.clone();
3564        let mut model = default_model.clone();
3565        let mut model_profile = model_profile_for_provider_model(&cfg, &provider, &model);
3566        let workspace = req.workspace.clone();
3567        let compaction_generation_at_start = self.compaction_generation(&req.thread_id);
3568        let mut transcript = self.transcript_for_turn(&req, &turn_id).await?;
3569        let mut compacted_this_turn =
3570            self.compaction_generation(&req.thread_id) != compaction_generation_at_start;
3571        let agent_swarm_mode_active = self
3572            .effective_agent_swarm_mode_for_thread(&req.thread_id)
3573            .await;
3574        let ultra_mode_active = self.effective_ultra_mode_for_thread(&req.thread_id).await;
3575        let thread_overrides = self.thread_turn_overrides(&req.thread_id).await?;
3576        let mut final_assistant_text = String::new();
3577        let mut final_provider_metadata = None;
3578        let mut exhausted_tool_rounds = true;
3579        let mut verification_gate =
3580            VerificationGateState::new(req.message.clone(), runtime_profile);
3581        let mut speed_policy = SpeedPolicyState::default();
3582        let mut reliability = TurnReliabilityState::default();
3583        let mut turn_usage = TokenUsage::default();
3584        // Overwritten on every inference step's Completed event so only the
3585        // terminal step's stop reason survives (mid-turn tool_use steps must
3586        // not leak onto turn/completed).
3587        let mut turn_finish_reason: Option<String> = None;
3588        let mut deadline_finalization_requested = false;
3589        let mut deadline_scoreable_completion_requested = false;
3590        let mut task_ledger_completion_reminders = 0_u8;
3591        let mut task_ledger_scoreable_checkpoints = 0_u8;
3592        let mut empty_tool_call_nudges_used = 0_u32;
3593        let mut provider_stream_retry_attempts = 0_u32;
3594        let mut context_limit_recovery_attempts = 0_u32;
3595        let mut routing_candidates = None;
3596        let routing_transcript_start = transcript.len().saturating_sub(1);
3597        let mut routing_escalations = 0_u32;
3598        let mut model_switch_summary_selection = None::<ModelSelection>;
3599
3600        'tool_rounds: for round_index in 0..MAX_TOOL_ROUNDS_PER_TURN {
3601            if let Some(deadline) = turn_deadline
3602                && deadline_expired(deadline)
3603            {
3604                self.fail_turn_due_to_deadline(&req.thread_id, &turn_id, deadline, &transcript)
3605                    .await?;
3606                return Ok(TurnRunOutcome::Stopped);
3607            }
3608            let steers = self.drain_turn_steers(&turn_id, &mut steering).await;
3609            self.append_steers(&req, &turn_id, &mut transcript, steers)
3610                .await?;
3611            if runtime_profile == RuntimeProfile::Eval
3612                && let Some(remaining) = crate::deadline_policy::should_start_finalization(
3613                    turn_deadline,
3614                    deadline_finalization_reserve,
3615                    deadline_finalization_requested || deadline_scoreable_completion_requested,
3616                )
3617            {
3618                if req.task_ledger_required
3619                    && task_ledger_completion_reminders < TASK_LEDGER_COMPLETION_REMINDER_LIMIT
3620                    && let Some(prompt) = task_ledger_completion_prompt(&transcript)
3621                {
3622                    task_ledger_completion_reminders += 1;
3623                    deadline_scoreable_completion_requested = true;
3624                    let item = TranscriptItem::UserMessage(UserMessage::text(
3625                        task_ledger_deadline_completion_prompt(
3626                            remaining,
3627                            deadline_finalization_reserve,
3628                            &prompt,
3629                        ),
3630                    ));
3631                    self.persist_turn_item(&req.thread_id, &turn_id, &item)
3632                        .await?;
3633                    transcript.push(item);
3634                    continue 'tool_rounds;
3635                } else {
3636                    self.start_deadline_finalization(
3637                        &req.thread_id,
3638                        &turn_id,
3639                        &mut transcript,
3640                        remaining,
3641                    )
3642                    .await?;
3643                    deadline_finalization_requested = true;
3644                }
3645            }
3646            if runtime_profile == RuntimeProfile::Eval
3647                && req.task_ledger_required
3648                && !deadline_finalization_requested
3649                && task_ledger_scoreable_checkpoints < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
3650                && let Some(remaining) = deadline_remaining_seconds(turn_deadline)
3651                && remaining <= TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS
3652                && remaining > deadline_finalization_reserve
3653                && let Some(prompt) = task_ledger_completion_prompt(&transcript)
3654            {
3655                task_ledger_scoreable_checkpoints += 1;
3656                let item = TranscriptItem::UserMessage(UserMessage::text(
3657                    task_ledger_scoreable_checkpoint_prompt(remaining, &prompt),
3658                ));
3659                self.persist_turn_item(&req.thread_id, &turn_id, &item)
3660                    .await?;
3661                transcript.push(item);
3662                continue 'tool_rounds;
3663            }
3664            if turn_inference_router.is_active() && routing_candidates.is_none() {
3665                routing_candidates =
3666                    Some(collect_inference_routing_candidates(&self.registry).await);
3667            }
3668            let routing_tools_model = model.clone();
3669            let routing_tools_provider = provider.clone();
3670            let routing_tools = self.filtered_tool_specs(
3671                &cfg,
3672                &model,
3673                &provider,
3674                model_profile.as_ref(),
3675                &thread_overrides.tool_allowlist,
3676                &thread_overrides.external_tools,
3677            );
3678            let prior_failures =
3679                transcript_failure_count_since(&transcript, routing_transcript_start)
3680                    .max(reliability.tool_failure_count())
3681                    .saturating_add(provider_stream_retry_attempts);
3682            let routing_selection = route_inference_selection(
3683                &self.registry,
3684                &turn_inference_router,
3685                InferenceRoutingRequest {
3686                    thread_id: &req.thread_id,
3687                    turn_id: &turn_id,
3688                    round_index: round_index as u32,
3689                    runtime_profile,
3690                    phase: speed_policy.phase(),
3691                    profile: turn_inference_router_profile.as_deref(),
3692                    default_selection: ModelSelection {
3693                        provider: default_provider.clone(),
3694                        model: default_model.clone(),
3695                    },
3696                    transcript: &transcript,
3697                    tools: &routing_tools,
3698                    candidates: routing_candidates.as_deref(),
3699                    prior_failures,
3700                    prior_escalations: routing_escalations,
3701                },
3702            )
3703            .await;
3704            if let Some(decision) = routing_selection.decision.clone() {
3705                if matches!(decision.outcome, InferenceRoutingOutcome::Escalated) {
3706                    routing_escalations = routing_escalations.saturating_add(1);
3707                }
3708                self.emit(RoderEvent::InferenceRoutingDecision(
3709                    InferenceRoutingDecisionEvent {
3710                        thread_id: req.thread_id.clone(),
3711                        turn_id: turn_id.clone(),
3712                        round_index: round_index as u32,
3713                        default_selection: ModelSelection {
3714                            provider: default_provider.clone(),
3715                            model: default_model.clone(),
3716                        },
3717                        selected_selection: routing_selection.selection.clone(),
3718                        decision,
3719                        timestamp: OffsetDateTime::now_utc(),
3720                    },
3721                ))
3722                .await;
3723            }
3724            provider = routing_selection.selection.provider.clone();
3725            model = routing_selection.selection.model.clone();
3726            let engine = self.engine_for(&provider)?;
3727            let capabilities = engine.capabilities();
3728            let tool_images = ToolImages::for_engine(engine.as_ref(), &model);
3729            model_profile = model_profile_for_provider_model(&cfg, &provider, &model);
3730            let tools = if capabilities.tool_calls {
3731                if model == routing_tools_model && provider == routing_tools_provider {
3732                    routing_tools.clone()
3733                } else {
3734                    self.filtered_tool_specs(
3735                        &cfg,
3736                        &model,
3737                        &provider,
3738                        model_profile.as_ref(),
3739                        &thread_overrides.tool_allowlist,
3740                        &thread_overrides.external_tools,
3741                    )
3742                }
3743            } else {
3744                Vec::new()
3745            };
3746            let parallel_tool_calls = parallel_tool_calls_for_model(&cfg, &model);
3747            let tool_choice = if tools.is_empty() {
3748                ToolChoice::None
3749            } else {
3750                ToolChoice::Auto
3751            };
3752            let summary_selection = ModelSelection {
3753                provider: provider.clone(),
3754                model: model.clone(),
3755            };
3756            if model_switch_summary_selection.as_ref() != Some(&summary_selection) {
3757                if let Some(summary) = model_switch_summary(
3758                    &transcript,
3759                    model_profile.as_ref(),
3760                    &provider,
3761                    &model,
3762                    &tools,
3763                ) {
3764                    let item = TranscriptItem::UserMessage(UserMessage::text(summary));
3765                    self.persist_turn_item(&req.thread_id, &turn_id, &item)
3766                        .await?;
3767                    transcript.push(item);
3768                }
3769                model_switch_summary_selection = Some(summary_selection);
3770            }
3771
3772            if !capabilities.image_input && transcript_has_images(&transcript) {
3773                self.fail_turn_with_error(
3774                    &req.thread_id,
3775                    &turn_id,
3776                    format!("provider {provider} does not support image input"),
3777                )
3778                .await?;
3779                return Ok(TurnRunOutcome::Stopped);
3780            }
3781            let compaction_generation_before = self.compaction_generation(&req.thread_id);
3782            transcript = self
3783                .compact_transcript_if_needed(
3784                    &req.thread_id,
3785                    &turn_id,
3786                    &provider,
3787                    &model,
3788                    transcript,
3789                    self.compaction_options_for_turn(&req.thread_id, !compacted_this_turn),
3790                )
3791                .await?;
3792            compacted_this_turn |=
3793                self.compaction_generation(&req.thread_id) != compaction_generation_before;
3794            if round_index == 0 {
3795                self.complete_context_assembly(&req, &turn_id, &transcript, tool_images)
3796                    .await;
3797            }
3798
3799            let speed_policy_decision =
3800                speed_policy.decision(runtime_profile, &model, &cfg.speed_policy);
3801            let request_reasoning = reasoning_from_decision(
3802                speed_policy_decision.as_ref(),
3803                routing_selection
3804                    .reasoning
3805                    .clone()
3806                    .unwrap_or_else(|| reasoning_for_model(&cfg, &model)),
3807            );
3808            self.active_turn_selections.write().await.insert(
3809                turn_id.clone(),
3810                ModelSelectionMode::manual(
3811                    provider.clone(),
3812                    model.clone(),
3813                    request_reasoning.level.clone(),
3814                ),
3815            );
3816            if let Some(limit) = reliability.record_model_call(
3817                &cfg.reliability,
3818                runtime_profile == RuntimeProfile::Interactive,
3819            ) {
3820                self.fail_turn_due_to_reliability_limit(
3821                    &req.thread_id,
3822                    &turn_id,
3823                    &provider,
3824                    &model,
3825                    limit,
3826                    &transcript,
3827                )
3828                .await?;
3829                return Ok(TurnRunOutcome::Stopped);
3830            }
3831            self.emit(RoderEvent::InferenceStarted(InferenceStarted {
3832                thread_id: req.thread_id.clone(),
3833                turn_id: turn_id.clone(),
3834                engine_id: engine.id(),
3835                model: ModelSelection {
3836                    provider: provider.clone(),
3837                    model: model.clone(),
3838                },
3839                reasoning: request_reasoning.clone(),
3840                speed_policy: speed_policy_decision.clone(),
3841                deadline_remaining_seconds: deadline_remaining_seconds(turn_deadline),
3842                timestamp: OffsetDateTime::now_utc(),
3843            }))
3844            .await;
3845
3846            let mut instructions = req.instructions.clone();
3847            if let Some(extra) = &thread_overrides.developer_instructions {
3848                instructions = apply_thread_developer_instructions(instructions, extra);
3849            }
3850            if let Some(context) = req.developer_context.as_deref() {
3851                instructions = apply_turn_developer_context(instructions, context);
3852            }
3853            let mut instructions = apply_runtime_profile(instructions, runtime_profile);
3854            if let Some(profile) = &model_profile {
3855                instructions = apply_model_instruction_overlay(instructions, profile);
3856            }
3857            if req.task_ledger_required
3858                && runtime_profile == RuntimeProfile::Eval
3859                && !transcript_has_task_ledger(&transcript)
3860            {
3861                instructions = apply_task_ledger_required(instructions);
3862            }
3863            let effective_policy_mode = self.effective_policy_mode_for_thread(&req.thread_id).await;
3864            if effective_policy_mode == PolicyMode::Plan {
3865                instructions = apply_plan_mode(instructions);
3866            }
3867            if agent_swarm_mode_active {
3868                instructions = apply_agent_swarm_mode(instructions);
3869            }
3870            // Ultra mode (any model) enables proactive multi-agent policy.
3871            // Codex Sol/Terra Ultra effort still does too without requiring the
3872            // mode flag. Models that advertise Ultra effort keep the
3873            // explicit-request-only policy on lower efforts when ultra mode is
3874            // off; other models get multi-agent policy only when ultra mode is on.
3875            let model_has_ultra_effort = model_supports_reasoning_effort(&model, REASONING_ULTRA);
3876            let effort_is_ultra = request_reasoning.level.as_deref() == Some(REASONING_ULTRA);
3877            let proactive_multi_agent = ultra_mode_active || effort_is_ultra;
3878            if ultra_mode_active || model_has_ultra_effort {
3879                instructions = apply_codex_multi_agent_mode(instructions, proactive_multi_agent);
3880            }
3881            let compaction_instructions = instructions.clone();
3882            instructions = self
3883                .goals
3884                .apply_goal_instructions(&req.thread_id, instructions, effective_policy_mode)
3885                .await?;
3886            let mut request_metadata = serde_json::json!({});
3887            if let Some(decision) = &speed_policy_decision {
3888                request_metadata["speedPolicy"] = serde_json::json!(decision);
3889            }
3890            if let Some(decision) = routing_selection.decision.as_ref() {
3891                request_metadata["inferenceRouting"] = serde_json::json!(decision);
3892            }
3893            if let Some(remaining) = deadline_remaining_seconds(turn_deadline) {
3894                request_metadata["deadlineRemainingSeconds"] = serde_json::json!(remaining);
3895            }
3896            if let Some(profile) = &model_profile {
3897                request_metadata["modelProfile"] = serde_json::json!({
3898                    "model": profile.model,
3899                    "providerFamily": profile.provider_family,
3900                    "editTool": profile.edit_tool,
3901                    "schemaPolicy": profile.schema_policy,
3902                    "instructionOverlay": profile.instruction_overlay,
3903                    "parallelToolCalls": profile.parallel_tool_calls,
3904                    "autoCompactTokenLimit": profile.auto_compact_token_limit,
3905                });
3906            }
3907            let task_ledger_required_this_round = req.task_ledger_required
3908                && runtime_profile == RuntimeProfile::Eval
3909                && !deadline_finalization_requested
3910                && !transcript_has_task_ledger(&transcript);
3911            let task_ledger_tools = (capabilities.tool_calls && task_ledger_required_this_round)
3912                .then(|| self.task_ledger_tool_specs(model_profile.as_ref()))
3913                .filter(|tools| !tools.is_empty());
3914            let request_tools = if deadline_finalization_requested {
3915                Vec::new()
3916            } else if let Some(ledger_tools) = &task_ledger_tools {
3917                ledger_tools.clone()
3918            } else {
3919                tools.clone()
3920            };
3921            // Tool guidance describes the tools this request can call, which is
3922            // fewer than `tools` on a deadline finalization or forced ledger round.
3923            instructions = apply_parallel_web_tools(
3924                instructions,
3925                request_tools.iter().map(|spec| spec.name.as_str()),
3926            );
3927            instructions = apply_browser_routing(
3928                instructions,
3929                request_tools.iter().map(|spec| spec.name.as_str()),
3930            );
3931            let request_tool_choice = if deadline_finalization_requested {
3932                ToolChoice::None
3933            } else if task_ledger_tools.is_some() {
3934                ToolChoice::Specific(TASK_LEDGER_TOOL_NAME.to_string())
3935            } else {
3936                tool_choice.clone()
3937            };
3938            if deadline_finalization_requested {
3939                request_metadata["deadlineFinalization"] = serde_json::json!({
3940                    "reserveSeconds": deadline_finalization_reserve,
3941                    "remainingSeconds": deadline_remaining_seconds(turn_deadline),
3942                });
3943            } else if deadline_scoreable_completion_requested {
3944                request_metadata["deadlineScoreableCompletion"] = serde_json::json!({
3945                    "reserveSeconds": deadline_finalization_reserve,
3946                    "remainingSeconds": deadline_remaining_seconds(turn_deadline),
3947                });
3948            }
3949            let mut eager_tools = EagerTools::new(
3950                self,
3951                &request_tools,
3952                &thread_overrides.external_tools,
3953                parallel_tool_calls,
3954            );
3955            let mut request_reliability: ReliabilityRequestPolicy = cfg.reliability.clone().into();
3956            request_reliability.provider_retry_max_attempts = request_reliability
3957                .provider_retry_max_attempts
3958                .saturating_sub(provider_stream_retry_attempts)
3959                .max(1);
3960            let request = AgentInferenceRequest {
3961                model: ModelSelection {
3962                    provider: provider.clone(),
3963                    model: model.clone(),
3964                },
3965                instructions,
3966                transcript: crate::tool_result_images::request_transcript(&transcript, tool_images),
3967                tools: request_tools,
3968                tool_choice: request_tool_choice,
3969                reasoning: request_reasoning,
3970                output: OutputConfig::default(),
3971                runtime: RuntimeHints {
3972                    auto_compact_token_limit: (provider != roder_api::catalog::PROVIDER_CODEX)
3973                        .then(|| server_side_compaction_threshold(&cfg, &model))
3974                        .flatten(),
3975                    profile: runtime_profile,
3976                    parallel_tool_calls: Some(parallel_tool_calls),
3977                    prompt_cache_key: Some(req.thread_id.clone()),
3978                    hosted_web_search: cfg.hosted_web_search.clone(),
3979                    tool_search: tool_search_for_provider_model(&cfg, &provider, &model),
3980                    speed_policy: speed_policy_decision,
3981                    reliability: Some(request_reliability),
3982                    deadline_remaining_seconds: deadline_remaining_seconds(turn_deadline),
3983                    service_tier: req.service_tier_override.clone(),
3984                    ..RuntimeHints::default()
3985                },
3986                metadata: request_metadata,
3987            };
3988
3989            let mut compaction_template = request.clone();
3990            compaction_template.instructions = compaction_instructions;
3991            compaction_template.transcript.clear();
3992            self.compaction_templates
3993                .write()
3994                .await
3995                .insert(req.thread_id.clone(), compaction_template);
3996
3997            let ctx = InferenceTurnContext {
3998                thread_id: &req.thread_id,
3999                turn_id: &turn_id,
4000                tool_executor: Some(std::sync::Arc::new(
4001                    crate::tool_execution::RuntimeTurnToolExecutor {
4002                        runtime: Arc::clone(self),
4003                        thread_id: req.thread_id.clone(),
4004                        turn_id: turn_id.clone(),
4005                        workspace: Some(workspace.clone()),
4006                        deadline: turn_deadline,
4007                    },
4008                )),
4009            };
4010            let stream_future = preemptible(engine.stream_turn(ctx, request), &mut steering);
4011            let mut stream = if let Some((deadline, timeout_action)) = inference_timeout_deadline(
4012                turn_deadline,
4013                runtime_profile,
4014                req.task_ledger_required,
4015                deadline_finalization_reserve,
4016                deadline_finalization_requested || deadline_scoreable_completion_requested,
4017                task_ledger_scoreable_checkpoints,
4018                &transcript,
4019            ) {
4020                match tokio::time::timeout_at(deadline_instant(deadline), stream_future).await {
4021                    Ok(stream) => match stream {
4022                        Ok(stream) => stream,
4023                        Err(err) => {
4024                            if err.is::<SamplingPreempted>() {
4025                                self.finish_sampling_cleanup(&turn_id).await?;
4026                                provider_stream_retry_attempts = 0;
4027                                continue 'tool_rounds;
4028                            }
4029                            self.finish_sampling_cleanup(&turn_id).await?;
4030                            if !deadline_finalization_requested
4031                                && self
4032                                    .retry_sampling_failure(
4033                                        SamplingRetry {
4034                                            thread_id: &req.thread_id,
4035                                            turn_id: &turn_id,
4036                                            provider: &provider,
4037                                            model: &model,
4038                                            config: &cfg.reliability,
4039                                            error: &err,
4040                                        },
4041                                        &mut provider_stream_retry_attempts,
4042                                        &mut steering,
4043                                    )
4044                                    .await
4045                            {
4046                                continue 'tool_rounds;
4047                            }
4048                            let error = err.to_string();
4049                            if self
4050                                .try_recover_from_context_limit(
4051                                    &req.thread_id,
4052                                    &turn_id,
4053                                    &provider,
4054                                    &model,
4055                                    &error,
4056                                    crate::compaction::is_context_overflow_failure(&err),
4057                                    &mut transcript,
4058                                    &mut compacted_this_turn,
4059                                    &mut context_limit_recovery_attempts,
4060                                )
4061                                .await?
4062                            {
4063                                continue 'tool_rounds;
4064                            }
4065                            return Err(err);
4066                        }
4067                    },
4068                    Err(_) => {
4069                        if runtime_profile == RuntimeProfile::Eval
4070                            && !deadline_finalization_requested
4071                        {
4072                            let remaining = deadline_remaining_seconds(turn_deadline).unwrap_or(0);
4073                            if timeout_action == InferenceTimeoutAction::ScoreableCheckpoint
4074                                && task_ledger_scoreable_checkpoints
4075                                    < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
4076                                && let Some(prompt) = task_ledger_completion_prompt(&transcript)
4077                            {
4078                                task_ledger_scoreable_checkpoints += 1;
4079                                let item = TranscriptItem::UserMessage(UserMessage::text(
4080                                    task_ledger_scoreable_checkpoint_prompt(remaining, &prompt),
4081                                ));
4082                                self.persist_turn_item(&req.thread_id, &turn_id, &item)
4083                                    .await?;
4084                                transcript.push(item);
4085                                continue 'tool_rounds;
4086                            }
4087                            self.start_deadline_finalization(
4088                                &req.thread_id,
4089                                &turn_id,
4090                                &mut transcript,
4091                                remaining,
4092                            )
4093                            .await?;
4094                            deadline_finalization_requested = true;
4095                            continue 'tool_rounds;
4096                        }
4097                        self.fail_turn_due_to_deadline(
4098                            &req.thread_id,
4099                            &turn_id,
4100                            deadline,
4101                            &transcript,
4102                        )
4103                        .await?;
4104                        return Ok(TurnRunOutcome::Stopped);
4105                    }
4106                }
4107            } else {
4108                match stream_future.await {
4109                    Ok(stream) => stream,
4110                    Err(err) => {
4111                        if err.is::<SamplingPreempted>() {
4112                            self.finish_sampling_cleanup(&turn_id).await?;
4113                            provider_stream_retry_attempts = 0;
4114                            continue 'tool_rounds;
4115                        }
4116                        self.finish_sampling_cleanup(&turn_id).await?;
4117                        if !deadline_finalization_requested
4118                            && self
4119                                .retry_sampling_failure(
4120                                    SamplingRetry {
4121                                        thread_id: &req.thread_id,
4122                                        turn_id: &turn_id,
4123                                        provider: &provider,
4124                                        model: &model,
4125                                        config: &cfg.reliability,
4126                                        error: &err,
4127                                    },
4128                                    &mut provider_stream_retry_attempts,
4129                                    &mut steering,
4130                                )
4131                                .await
4132                        {
4133                            continue 'tool_rounds;
4134                        }
4135                        let error = err.to_string();
4136                        if self
4137                            .try_recover_from_context_limit(
4138                                &req.thread_id,
4139                                &turn_id,
4140                                &provider,
4141                                &model,
4142                                &error,
4143                                crate::compaction::is_context_overflow_failure(&err),
4144                                &mut transcript,
4145                                &mut compacted_this_turn,
4146                                &mut context_limit_recovery_attempts,
4147                            )
4148                            .await?
4149                        {
4150                            continue 'tool_rounds;
4151                        }
4152                        return Err(err);
4153                    }
4154                }
4155            };
4156            let mut sampling_output = SamplingOutput::default();
4157            let output_context = OutputContext {
4158                thread_id: &req.thread_id,
4159                turn_id: &turn_id,
4160                profile: model_profile.as_ref(),
4161                provider: &provider,
4162                model: &model,
4163            };
4164            let mut assistant_text = String::new();
4165            let mut phase_messages = Vec::<AssistantMessage>::new();
4166            let mut reasoning_text = String::new();
4167            let mut replayed_tool_call = false;
4168            let mut tool_calls = Vec::new();
4169            let mut patch_progress = PatchProgressTracker::default();
4170            let mut provider_metadata = None;
4171            let mut provider_requests_continuation = false;
4172            // Captured from mid-stream Compaction(completed) events. Codex/OpenAI
4173            // sometimes abort the SSE stream after emitting the compaction item
4174            // ("error decoding response body") before response.completed, so we
4175            // must not rely solely on ProviderMetadata for the boundary.
4176            let mut stream_compaction_item: Option<serde_json::Value> = None;
4177
4178            loop {
4179                let next_future = async {
4180                    loop {
4181                        tokio::select! {
4182                            biased;
4183                            _ = wait_for_steer(&mut steering) => break Some(Err(SamplingPreempted.into())),
4184                            result = eager_tools.poll_result(), if eager_tools.pending() => {
4185                                if let Err(error) = result { break Some(Err(error)); }
4186                            }
4187                            next = stream.next() => break next,
4188                        }
4189                    }
4190                };
4191                let next = if let Some((deadline, timeout_action)) = inference_timeout_deadline(
4192                    turn_deadline,
4193                    runtime_profile,
4194                    req.task_ledger_required,
4195                    deadline_finalization_reserve,
4196                    deadline_finalization_requested || deadline_scoreable_completion_requested,
4197                    task_ledger_scoreable_checkpoints,
4198                    &transcript,
4199                ) {
4200                    match tokio::time::timeout_at(deadline_instant(deadline), next_future).await {
4201                        Ok(next) => next,
4202                        Err(_) => {
4203                            if runtime_profile == RuntimeProfile::Eval
4204                                && !deadline_finalization_requested
4205                            {
4206                                let remaining =
4207                                    deadline_remaining_seconds(turn_deadline).unwrap_or(0);
4208                                if timeout_action == InferenceTimeoutAction::ScoreableCheckpoint
4209                                    && task_ledger_scoreable_checkpoints
4210                                        < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
4211                                    && let Some(prompt) = task_ledger_completion_prompt(&transcript)
4212                                {
4213                                    task_ledger_scoreable_checkpoints += 1;
4214                                    let item = TranscriptItem::UserMessage(UserMessage::text(
4215                                        task_ledger_scoreable_checkpoint_prompt(remaining, &prompt),
4216                                    ));
4217                                    self.persist_turn_item(&req.thread_id, &turn_id, &item)
4218                                        .await?;
4219                                    transcript.push(item);
4220                                    continue 'tool_rounds;
4221                                }
4222                                self.start_deadline_finalization(
4223                                    &req.thread_id,
4224                                    &turn_id,
4225                                    &mut transcript,
4226                                    remaining,
4227                                )
4228                                .await?;
4229                                deadline_finalization_requested = true;
4230                                continue 'tool_rounds;
4231                            }
4232                            self.fail_turn_due_to_deadline(
4233                                &req.thread_id,
4234                                &turn_id,
4235                                deadline,
4236                                &transcript,
4237                            )
4238                            .await?;
4239                            return Ok(TurnRunOutcome::Stopped);
4240                        }
4241                    }
4242                } else {
4243                    next_future.await
4244                };
4245                let res = next.unwrap_or_else(|| {
4246                    Err(roder_api::provider_error::ProviderFailure::new(
4247                        roder_api::provider_error::ProviderFailureKind::StreamInterrupted,
4248                        "provider stream ended before terminal completion",
4249                    )
4250                    .into())
4251                });
4252                let event = match res {
4253                    Ok(event) => event,
4254                    Err(err) => {
4255                        drop(stream);
4256                        eager_tools.drain().await?;
4257                        for result in eager_tools.take_results(&tool_calls) {
4258                            verification_gate.record_tool_result(&result);
4259                            transcript.push(TranscriptItem::ToolResult(result));
4260                        }
4261                        self.finish_sampling_cleanup(&turn_id).await?;
4262                        if err.is::<SamplingPreempted>() {
4263                            provider_stream_retry_attempts = 0;
4264                            continue 'tool_rounds;
4265                        }
4266                        if !deadline_finalization_requested
4267                            && self
4268                                .retry_sampling_failure(
4269                                    SamplingRetry {
4270                                        thread_id: &req.thread_id,
4271                                        turn_id: &turn_id,
4272                                        provider: &provider,
4273                                        model: &model,
4274                                        config: &cfg.reliability,
4275                                        error: &err,
4276                                    },
4277                                    &mut provider_stream_retry_attempts,
4278                                    &mut steering,
4279                                )
4280                                .await
4281                        {
4282                            continue 'tool_rounds;
4283                        }
4284                        let error = err.to_string();
4285                        if self
4286                            .try_recover_from_context_limit(
4287                                &req.thread_id,
4288                                &turn_id,
4289                                &provider,
4290                                &model,
4291                                &error,
4292                                crate::compaction::is_context_overflow_failure(&err),
4293                                &mut transcript,
4294                                &mut compacted_this_turn,
4295                                &mut context_limit_recovery_attempts,
4296                            )
4297                            .await?
4298                        {
4299                            continue 'tool_rounds;
4300                        }
4301                        return Err(err);
4302                    }
4303                };
4304
4305                let inference_timestamp = OffsetDateTime::now_utc();
4306                self.emit(RoderEvent::InferenceEventReceived(InferenceEventReceived {
4307                    thread_id: req.thread_id.clone(),
4308                    turn_id: turn_id.clone(),
4309                    event: event.clone(),
4310                    timestamp: inference_timestamp,
4311                }))
4312                .await;
4313
4314                match event {
4315                    InferenceEvent::MessageDelta(delta) => {
4316                        if !delta.text.trim().is_empty() {
4317                            self.goals.record_activity(&turn_id).await;
4318                        }
4319                        if let Some((team_id, member)) =
4320                            self.teams.member_for_thread(&req.thread_id).await
4321                        {
4322                            self.emit(RoderEvent::TeamMemberMessageDelta(TeamMemberMessageDelta {
4323                                team_id,
4324                                member_id: member.id,
4325                                member_thread_id: req.thread_id.clone(),
4326                                turn_id: turn_id.clone(),
4327                                delta: delta.text.clone(),
4328                                timestamp: OffsetDateTime::now_utc(),
4329                            }))
4330                            .await;
4331                        }
4332                        if is_final_answer_phase(delta.phase.as_deref()) {
4333                            assistant_text.push_str(&delta.text);
4334                        } else if let Some(last) = phase_messages.last_mut()
4335                            && last.phase == delta.phase
4336                        {
4337                            last.text.push_str(&delta.text);
4338                        } else {
4339                            phase_messages.push(AssistantMessage {
4340                                text: delta.text,
4341                                phase: delta.phase,
4342                            });
4343                        }
4344                    }
4345                    InferenceEvent::ReasoningDelta(delta) => reasoning_text.push_str(&delta.text),
4346                    InferenceEvent::ToolCallCompleted(call) => {
4347                        self.goals.record_activity(&turn_id).await;
4348                        if completed_call_replayed(&transcript, &call)? {
4349                            replayed_tool_call = true;
4350                            continue;
4351                        }
4352                        if let Some(progress) = patch_progress.complete(
4353                            &req.thread_id,
4354                            &turn_id,
4355                            &call.id,
4356                            &call.name,
4357                            &call.arguments,
4358                        ) {
4359                            self.emit(RoderEvent::PatchProgress(progress)).await;
4360                        }
4361                        if !tool_calls
4362                            .iter()
4363                            .any(|previous: &ToolCallCompleted| previous.id == call.id)
4364                        {
4365                            eager_tools
4366                                .schedule(
4367                                    self,
4368                                    &output_context,
4369                                    &call,
4370                                    workspace.as_str(),
4371                                    turn_deadline,
4372                                    &mut transcript,
4373                                )
4374                                .await?;
4375                            tool_calls.push(call);
4376                        }
4377                    }
4378                    InferenceEvent::OutputItemCompleted(item) => {
4379                        sampling_output
4380                            .complete_item(self, &output_context, item, &mut transcript)
4381                            .await?;
4382                    }
4383                    InferenceEvent::Failed(failure) => {
4384                        speed_policy.record_failure();
4385                        let error = failure.message;
4386                        if self
4387                            .try_recover_from_context_limit(
4388                                &req.thread_id,
4389                                &turn_id,
4390                                &provider,
4391                                &model,
4392                                &error,
4393                                false,
4394                                &mut transcript,
4395                                &mut compacted_this_turn,
4396                                &mut context_limit_recovery_attempts,
4397                            )
4398                            .await?
4399                        {
4400                            continue 'tool_rounds;
4401                        }
4402                        self.persist_turn_item(
4403                            &req.thread_id,
4404                            &turn_id,
4405                            &TranscriptItem::Error(ErrorRecord {
4406                                message: error.clone(),
4407                            }),
4408                        )
4409                        .await?;
4410                        self.emit(RoderEvent::TurnFailed(TurnFailed {
4411                            thread_id: req.thread_id.clone(),
4412                            turn_id: turn_id.clone(),
4413                            error: error.clone(),
4414                            error_kind: None,
4415                            usage: None,
4416                            timestamp: OffsetDateTime::now_utc(),
4417                        }))
4418                        .await;
4419                        self.complete_team_member_turn_with_result(
4420                            &req.thread_id,
4421                            &turn_id,
4422                            TeamMemberStatus::Failed,
4423                            (!assistant_text.trim().is_empty()).then(|| assistant_text.clone()),
4424                            Some(error),
4425                        )
4426                        .await?;
4427                        return Ok(TurnRunOutcome::Stopped);
4428                    }
4429                    InferenceEvent::Usage(usage) => {
4430                        self.record_goal_token_usage(
4431                            &req.thread_id,
4432                            &turn_id,
4433                            usage.total_tokens as i64,
4434                        )
4435                        .await?;
4436                        turn_usage.add_assign(&usage);
4437                    }
4438                    InferenceEvent::Completed(metadata) => {
4439                        sampling_output
4440                            .finish(
4441                                self,
4442                                &output_context,
4443                                &assistant_text,
4444                                &phase_messages,
4445                                &reasoning_text,
4446                                &mut transcript,
4447                            )
4448                            .await?;
4449                        provider_stream_retry_attempts = 0;
4450                        turn_finish_reason = metadata
4451                            .stop_reason
4452                            .as_deref()
4453                            .map(finish_reason_from_stop_reason);
4454                        break;
4455                    }
4456                    InferenceEvent::Compaction(progress) => {
4457                        // Persist the boundary as soon as the provider emits a
4458                        // completed compaction item. ChatGPT Codex often aborts
4459                        // the SSE stream right after ("error decoding response
4460                        // body"), so waiting for ProviderMetadata loses the
4461                        // boundary and every subsequent request re-compacts.
4462                        if progress.status == "completed"
4463                            && let Some(item) = progress.item
4464                        {
4465                            let already = stream_compaction_item
4466                                .as_ref()
4467                                .and_then(|existing| {
4468                                    existing.get("id").and_then(serde_json::Value::as_str)
4469                                })
4470                                .zip(item.get("id").and_then(serde_json::Value::as_str))
4471                                .is_some_and(|(a, b)| a == b);
4472                            if !already {
4473                                stream_compaction_item = Some(item.clone());
4474                                let boundary = TranscriptItem::ProviderMetadata(
4475                                    crate::compaction::provider_metadata_from_compaction_item(item),
4476                                );
4477                                self.persist_turn_item(&req.thread_id, &turn_id, &boundary)
4478                                    .await?;
4479                                transcript.push(boundary);
4480                                transcript =
4481                                    crate::compaction::trim_to_last_compaction_boundary(transcript);
4482                                compacted_this_turn = true;
4483                            }
4484                        }
4485                    }
4486                    InferenceEvent::HostedToolCallStarted(_)
4487                    | InferenceEvent::HostedToolCallCompleted(_) => {}
4488                    InferenceEvent::ToolCallStarted(call) => {
4489                        patch_progress.start(&call.id, &call.name)
4490                    }
4491                    InferenceEvent::ToolCallDelta(delta) => {
4492                        if let Some(progress) = patch_progress.delta(
4493                            &req.thread_id,
4494                            &turn_id,
4495                            &delta.id,
4496                            &delta.arguments_delta,
4497                        ) {
4498                            self.emit(RoderEvent::PatchProgress(progress)).await;
4499                        }
4500                    }
4501                    InferenceEvent::ProviderMetadata(mut metadata) => {
4502                        if metadata.get("kind").and_then(serde_json::Value::as_str)
4503                            == Some("reliability_retry_attempt")
4504                        {
4505                            provider_stream_retry_attempts =
4506                                provider_stream_retry_attempts.saturating_add(1);
4507                        }
4508                        provider_requests_continuation |= metadata
4509                            .get("end_turn")
4510                            .and_then(serde_json::Value::as_bool)
4511                            == Some(false);
4512                        if let Some(compaction_item) = stream_compaction_item.as_ref() {
4513                            let _ = crate::compaction::ensure_provider_metadata_compaction(
4514                                &mut metadata,
4515                                compaction_item,
4516                            );
4517                        }
4518                        if metadata.get("output").is_none() {
4519                            let item = TranscriptItem::ProviderMetadata(metadata);
4520                            self.persist_turn_item(&req.thread_id, &turn_id, &item)
4521                                .await?;
4522                            transcript.push(item);
4523                        } else {
4524                            provider_metadata = Some(metadata);
4525                        }
4526                    }
4527                }
4528            }
4529
4530            speed_policy.record_model_output(
4531                !assistant_text.is_empty() || !phase_messages.is_empty(),
4532                tool_calls.len(),
4533            );
4534            if tool_calls.is_empty() {
4535                if provider_requests_continuation || replayed_tool_call {
4536                    if let Some(metadata) = provider_metadata {
4537                        let item = TranscriptItem::ProviderMetadata(metadata);
4538                        self.persist_turn_item(&req.thread_id, &turn_id, &item)
4539                            .await?;
4540                        transcript.push(item);
4541                    }
4542                    continue 'tool_rounds;
4543                }
4544                let steers = self.drain_turn_steers(&turn_id, &mut steering).await;
4545                if !steers.is_empty() {
4546                    if let Some(metadata) = provider_metadata {
4547                        let had_provider_compaction =
4548                            crate::compaction::provider_metadata_has_compaction(&metadata);
4549                        let item = TranscriptItem::ProviderMetadata(metadata);
4550                        self.persist_turn_item(&req.thread_id, &turn_id, &item)
4551                            .await?;
4552                        transcript.push(item);
4553                        if had_provider_compaction {
4554                            // Server-side compaction replaced the prior window;
4555                            // drop pre-boundary items so the next request does
4556                            // not re-send (and re-compact) the full history.
4557                            transcript =
4558                                crate::compaction::trim_to_last_compaction_boundary(transcript);
4559                            compacted_this_turn = true;
4560                        }
4561                    }
4562                    self.append_steers(&req, &turn_id, &mut transcript, steers)
4563                        .await?;
4564                    continue;
4565                }
4566                if !deadline_finalization_requested
4567                    && req.task_ledger_required
4568                    && runtime_profile == RuntimeProfile::Eval
4569                    && task_ledger_completion_reminders < TASK_LEDGER_COMPLETION_REMINDER_LIMIT
4570                    && (!assistant_text.trim().is_empty() || !phase_messages.is_empty())
4571                    && let Some(prompt) = task_ledger_completion_prompt(&transcript)
4572                {
4573                    task_ledger_completion_reminders += 1;
4574                    let item = TranscriptItem::UserMessage(UserMessage::text(prompt));
4575                    self.persist_turn_item(&req.thread_id, &turn_id, &item)
4576                        .await?;
4577                    transcript.push(item);
4578                    continue;
4579                }
4580                if !deadline_finalization_requested
4581                    && let Some(prompt) = verification_gate.blocking_prompt()
4582                {
4583                    speed_policy.record_verification_required();
4584                    self.emit(RoderEvent::VerificationRequired(VerificationRequired {
4585                        thread_id: req.thread_id.clone(),
4586                        turn_id: turn_id.clone(),
4587                        reason: verification_gate.reason(),
4588                        changed_files: verification_gate.changed_files(),
4589                        tool_evidence: verification_gate.tool_evidence.clone(),
4590                        tests_run: verification_gate.tests_run.clone(),
4591                        open_gaps: verification_gate.open_gaps.clone(),
4592                        timestamp: OffsetDateTime::now_utc(),
4593                    }))
4594                    .await;
4595                    let item = TranscriptItem::UserMessage(UserMessage::text(prompt));
4596                    self.persist_turn_item(&req.thread_id, &turn_id, &item)
4597                        .await?;
4598                    transcript.push(item);
4599                    continue;
4600                }
4601                // Loop persistence nudge: in non-interactive/eval runs, when the
4602                // model returns a final message with no tool calls, gently prompt
4603                // it to keep going (bounded by `empty_tool_call_nudges`) before
4604                // ending the turn. Gated to non-interactive/eval so interactive
4605                // chat completions are never delayed, and only fires when the
4606                // model actually produced a final answer to re-examine.
4607                if !deadline_finalization_requested
4608                    && runtime_profile != RuntimeProfile::Interactive
4609                    && cfg.reliability.empty_tool_call_nudges > 0
4610                    && empty_tool_call_nudges_used < cfg.reliability.empty_tool_call_nudges
4611                    && (!assistant_text.trim().is_empty() || !phase_messages.is_empty())
4612                {
4613                    empty_tool_call_nudges_used += 1;
4614                    let item = TranscriptItem::UserMessage(UserMessage::text(
4615                        RELIABILITY_CONTINUATION_PROMPT.to_string(),
4616                    ));
4617                    self.persist_turn_item(&req.thread_id, &turn_id, &item)
4618                        .await?;
4619                    transcript.push(item);
4620                    continue;
4621                }
4622                if deadline_finalization_requested
4623                    && assistant_text.trim().is_empty()
4624                    && phase_messages.is_empty()
4625                {
4626                    assistant_text = format!(
4627                        "Deadline finalization completed without model text. {}",
4628                        turn_partial_result(&transcript)
4629                    );
4630                }
4631                final_assistant_text = assistant_text;
4632                final_provider_metadata = provider_metadata;
4633                exhausted_tool_rounds = false;
4634                break;
4635            }
4636
4637            if let Some(metadata) = provider_metadata {
4638                let had_provider_compaction =
4639                    crate::compaction::provider_metadata_has_compaction(&metadata);
4640                let item = TranscriptItem::ProviderMetadata(metadata);
4641                self.persist_turn_item(&req.thread_id, &turn_id, &item)
4642                    .await?;
4643                transcript.push(item);
4644                if had_provider_compaction {
4645                    // Server-side compaction replaced the prior window; drop
4646                    // pre-boundary items so subsequent tool rounds use the
4647                    // compact window as the next input.
4648                    transcript = crate::compaction::trim_to_last_compaction_boundary(transcript);
4649                    compacted_this_turn = true;
4650                }
4651            }
4652            eager_tools.drain().await?;
4653            for call in &tool_calls {
4654                if eager_tools.scheduled(&call.id) {
4655                    continue;
4656                }
4657                let tool_item = TranscriptItem::ToolCall(ToolCallRecord {
4658                    id: call.id.clone(),
4659                    name: call.name.clone(),
4660                    arguments: call.arguments.clone(),
4661                });
4662                self.persist_turn_item(&req.thread_id, &turn_id, &tool_item)
4663                    .await?;
4664                transcript.push(tool_item);
4665                self.persist_model_profile_segment(
4666                    &req.thread_id,
4667                    &turn_id,
4668                    model_profile.as_ref(),
4669                    &provider,
4670                    &model,
4671                    "tool_call",
4672                )
4673                .await?;
4674            }
4675            if let Some(deadline) = turn_deadline
4676                && deadline_expired(deadline)
4677            {
4678                self.fail_turn_due_to_deadline(&req.thread_id, &turn_id, deadline, &transcript)
4679                    .await?;
4680                return Ok(TurnRunOutcome::Stopped);
4681            }
4682            let mut results = eager_tools.take_results(&tool_calls);
4683            let remaining_calls = tool_calls
4684                .into_iter()
4685                .filter(|call| !eager_tools.scheduled(&call.id))
4686                .collect();
4687            results.extend(
4688                self.route_tool_calls(
4689                    &req.thread_id,
4690                    &turn_id,
4691                    remaining_calls,
4692                    parallel_tool_calls,
4693                    Some(workspace.as_str()),
4694                    turn_deadline,
4695                )
4696                .await?,
4697            );
4698            let reliability_limit = reliability.record_tool_results(
4699                &cfg.reliability,
4700                &results,
4701                runtime_profile == RuntimeProfile::Interactive,
4702            );
4703            for result in results {
4704                verification_gate.record_tool_result(&result);
4705                transcript.push(TranscriptItem::ToolResult(result));
4706                self.persist_model_profile_segment(
4707                    &req.thread_id,
4708                    &turn_id,
4709                    model_profile.as_ref(),
4710                    &provider,
4711                    &model,
4712                    "tool_result",
4713                )
4714                .await?;
4715            }
4716            if let Some(limit) = reliability_limit {
4717                if limit.decision == ReliabilityLimitDecision::RequestContinuation {
4718                    // Loop persistence: rather than ending the turn, reset the
4719                    // consecutive-failure counter, nudge the model to keep going,
4720                    // and continue the round loop. Bounded by the per-turn tool
4721                    // failure ceiling, `max_model_calls_per_turn`, and
4722                    // `MAX_TOOL_ROUNDS_PER_TURN`.
4723                    self.record_reliability_limit_continuation(
4724                        &req.thread_id,
4725                        &turn_id,
4726                        &provider,
4727                        &model,
4728                        limit,
4729                    )
4730                    .await;
4731                    reliability.reset_consecutive_failures();
4732                    let nudge = TranscriptItem::UserMessage(UserMessage::text(
4733                        RELIABILITY_CONTINUATION_PROMPT.to_string(),
4734                    ));
4735                    self.persist_turn_item(&req.thread_id, &turn_id, &nudge)
4736                        .await?;
4737                    transcript.push(nudge);
4738                } else {
4739                    self.fail_turn_due_to_reliability_limit(
4740                        &req.thread_id,
4741                        &turn_id,
4742                        &provider,
4743                        &model,
4744                        limit,
4745                        &transcript,
4746                    )
4747                    .await?;
4748                    return Ok(TurnRunOutcome::Stopped);
4749                }
4750            }
4751            let compaction_generation_before = self.compaction_generation(&req.thread_id);
4752            transcript = self
4753                .compact_transcript_if_needed(
4754                    &req.thread_id,
4755                    &turn_id,
4756                    &provider,
4757                    &model,
4758                    transcript,
4759                    self.compaction_options_for_turn(&req.thread_id, !compacted_this_turn),
4760                )
4761                .await?;
4762            compacted_this_turn |=
4763                self.compaction_generation(&req.thread_id) != compaction_generation_before;
4764        }
4765
4766        if exhausted_tool_rounds {
4767            let message =
4768                format!("tool call limit reached after {MAX_TOOL_ROUNDS_PER_TURN} rounds");
4769            self.persist_turn_item(
4770                &req.thread_id,
4771                &turn_id,
4772                &TranscriptItem::Error(ErrorRecord {
4773                    message: message.clone(),
4774                }),
4775            )
4776            .await?;
4777            self.emit(RoderEvent::TurnFailed(TurnFailed {
4778                thread_id: req.thread_id.clone(),
4779                turn_id: turn_id.clone(),
4780                error: message.clone(),
4781                error_kind: None,
4782                usage: None,
4783                timestamp: OffsetDateTime::now_utc(),
4784            }))
4785            .await;
4786            self.complete_team_member_turn_with_result(
4787                &req.thread_id,
4788                &turn_id,
4789                TeamMemberStatus::Failed,
4790                None,
4791                Some(message),
4792            )
4793            .await?;
4794            return Ok(TurnRunOutcome::Stopped);
4795        }
4796
4797        if let Some(metadata) = final_provider_metadata {
4798            self.persist_turn_item(
4799                &req.thread_id,
4800                &turn_id,
4801                &TranscriptItem::ProviderMetadata(metadata),
4802            )
4803            .await?;
4804        }
4805
4806        let completed_usage = (!turn_usage.is_empty()).then_some(turn_usage.clone());
4807        self.record_thread_usage_metadata(&req.thread_id, &turn_usage)
4808            .await?;
4809        let (cleanup, ownership) = self.await_provider_turn_cleanup(&turn_id).await;
4810        self.record_turn_lifecycle_with_ownership(
4811            req.thread_id.clone(),
4812            turn_id.clone(),
4813            TurnLifecycleState::Completed,
4814            cleanup,
4815            None,
4816            ownership,
4817        )
4818        .await;
4819        self.emit(RoderEvent::TurnCompleted(TurnCompleted {
4820            thread_id: req.thread_id.clone(),
4821            turn_id: turn_id.clone(),
4822            usage: completed_usage,
4823            finish_reason: turn_finish_reason,
4824            timestamp: OffsetDateTime::now_utc(),
4825        }))
4826        .await;
4827        self.complete_team_member_turn_with_result(
4828            &req.thread_id,
4829            &turn_id,
4830            TeamMemberStatus::Completed,
4831            (!final_assistant_text.is_empty()).then_some(final_assistant_text),
4832            None,
4833        )
4834        .await?;
4835        Ok(TurnRunOutcome::Completed)
4836    }
4837
4838    async fn route_tool_calls(
4839        self: &Arc<Self>,
4840        thread_id: &ThreadId,
4841        turn_id: &TurnId,
4842        calls: Vec<ToolCallCompleted>,
4843        parallel: bool,
4844        workspace: Option<&str>,
4845        deadline: Option<OffsetDateTime>,
4846    ) -> anyhow::Result<Vec<ToolResultRecord>> {
4847        // Enforce the agent_swarm exclusivity rule (roadmap 104, Task 2):
4848        // `agent_swarm` must be the only tool call in a model response. A mixed
4849        // or multi-swarm batch is denied wholesale with actionable retry text so
4850        // the model re-issues `agent_swarm` by itself, and every tool_call_id
4851        // still gets a response (keeping the chat-completions transcript valid).
4852        if let Some(violation) = roder_api::subagents::agent_swarm_batch_violation(
4853            calls.iter().map(|call| call.name.as_str()),
4854        ) {
4855            let message = violation.deny_message();
4856            return Ok(calls
4857                .into_iter()
4858                .map(|call| ToolResultRecord {
4859                    id: call.id,
4860                    name: Some(call.name),
4861                    result: message.clone(),
4862                    display_payload: None,
4863                    is_error: true,
4864                })
4865                .collect());
4866        }
4867        // Emit swarm lifecycle events on the bus (roadmap 104, Task 1) so any
4868        // app-server/SDK/TUI client can observe a swarm as a whole; per-child
4869        // progress flows through the existing Subagent* trace events.
4870        let swarm_call = (calls.len() == 1
4871            && calls[0].name == roder_api::subagents::AGENT_SWARM_TOOL_NAME)
4872            .then(|| (calls[0].id.clone(), calls[0].arguments.clone()));
4873        if let Some((tool_id, args)) = &swarm_call {
4874            self.emit(RoderEvent::AgentSwarmStarted(
4875                roder_api::subagents::AgentSwarmStarted {
4876                    thread_id: thread_id.clone(),
4877                    turn_id: turn_id.clone(),
4878                    tool_id: tool_id.clone(),
4879                    child_count: agent_swarm_child_count(args),
4880                    timestamp: OffsetDateTime::now_utc(),
4881                },
4882            ))
4883            .await;
4884        }
4885
4886        let force_sequential = calls
4887            .iter()
4888            .any(|call| crate::agent_control_tools::is_agent_control_tool(&call.name));
4889        let results =
4890            if parallel && !force_sequential {
4891                try_join_all(calls.into_iter().map(|call| {
4892                    self.route_tool_call(thread_id, turn_id, call, workspace, deadline)
4893                }))
4894                .await
4895            } else {
4896                let mut results = Vec::with_capacity(calls.len());
4897                for call in calls {
4898                    results.push(
4899                        self.route_tool_call(thread_id, turn_id, call, workspace, deadline)
4900                            .await?,
4901                    );
4902                }
4903                Ok(results)
4904            }?;
4905
4906        if let Some((tool_id, _)) = &swarm_call
4907            && let Some(result) = results.iter().find(|result| &result.id == tool_id)
4908            && let Some((completed, failed, aborted)) = parse_swarm_counts(&result.result)
4909        {
4910            self.emit(RoderEvent::AgentSwarmCompleted(
4911                roder_api::subagents::AgentSwarmCompleted {
4912                    thread_id: thread_id.clone(),
4913                    turn_id: turn_id.clone(),
4914                    tool_id: tool_id.clone(),
4915                    completed,
4916                    failed,
4917                    aborted,
4918                    timestamp: OffsetDateTime::now_utc(),
4919                },
4920            ))
4921            .await;
4922        }
4923
4924        Ok(results)
4925    }
4926
4927    /// On provider context/prompt-length failures, force-compact (and if needed
4928    /// strip the last bulky item) then retry the current tool round instead of
4929    /// failing the turn. Bounded so a permanently oversized suffix still stops.
4930    async fn try_recover_from_context_limit(
4931        &self,
4932        thread_id: &ThreadId,
4933        turn_id: &TurnId,
4934        provider: &str,
4935        model: &str,
4936        error: &str,
4937        typed_overflow: bool,
4938        transcript: &mut Vec<TranscriptItem>,
4939        compacted_this_turn: &mut bool,
4940        recovery_attempts: &mut u32,
4941    ) -> anyhow::Result<bool> {
4942        const MAX_CONTEXT_LIMIT_RECOVERY_ATTEMPTS: u32 = 2;
4943        if !typed_overflow && !crate::compaction::is_context_limit_failure_message(error) {
4944            return Ok(false);
4945        }
4946        if *recovery_attempts >= MAX_CONTEXT_LIMIT_RECOVERY_ATTEMPTS {
4947            return Ok(false);
4948        }
4949        *recovery_attempts = recovery_attempts.saturating_add(1);
4950
4951        let error_item = TranscriptItem::Error(ErrorRecord {
4952            message: error.to_string(),
4953        });
4954        self.persist_turn_item(thread_id, turn_id, &error_item)
4955            .await?;
4956        transcript.push(error_item);
4957
4958        // After the first failed recovery, drop the item ahead of the latest
4959        // user/error so a second compact can succeed when the suffix itself is
4960        // still oversized (e.g. a huge tool result right before the prompt).
4961        if *recovery_attempts > 1 {
4962            *transcript =
4963                crate::compaction::strip_last_message_before_error(std::mem::take(transcript));
4964        }
4965
4966        let force_options = crate::compaction::CompactionOptions {
4967            allow_repeat: true,
4968            force: true,
4969            hysteresis_baseline: None,
4970            preserve_hint: Some(
4971                "Recover from a provider context/prompt-length limit; preserve the active user goal and recent tool outcomes."
4972                    .to_string(),
4973            ),
4974        };
4975        let before_len = transcript.len();
4976        let tool_images = self.tool_images(provider, model);
4977        let before_tokens = crate::compaction::estimate_prompt_tokens(transcript, tool_images);
4978        *transcript = self
4979            .compact_transcript_if_needed(
4980                thread_id,
4981                turn_id,
4982                provider,
4983                model,
4984                std::mem::take(transcript),
4985                force_options,
4986            )
4987            .await?;
4988        let after_tokens = crate::compaction::estimate_prompt_tokens(transcript, tool_images);
4989        *compacted_this_turn = *compacted_this_turn
4990            || transcript
4991                .iter()
4992                .any(crate::compaction::is_compaction_boundary);
4993
4994        self.emit(RoderEvent::ReliabilityRetryRecorded(
4995            ReliabilityRetryRecorded {
4996                context: ReliabilityContext {
4997                    thread_id: thread_id.clone(),
4998                    turn_id: turn_id.clone(),
4999                    provider: Some(provider.to_string()),
5000                    model: Some(model.to_string()),
5001                    ..ReliabilityContext::default()
5002                },
5003                error_class: ReliabilityErrorClass::ProviderError,
5004                decision: ReliabilityRetryDecision::Retry,
5005                attempt: *recovery_attempts,
5006                max_attempts: MAX_CONTEXT_LIMIT_RECOVERY_ATTEMPTS,
5007                delay_ms: Some(0),
5008                details: ReliabilityDetails::redacted(format!(
5009                    "context_limit_recovery: compact {before_tokens}->{after_tokens} tokens (items {before_len}->{}); {error}",
5010                    transcript.len()
5011                )),
5012                timestamp: OffsetDateTime::now_utc(),
5013            },
5014        ))
5015        .await;
5016
5017        // If force-compact did not shrink anything and we still have budget,
5018        // strip once more so the next round is not identical.
5019        if after_tokens >= before_tokens && *recovery_attempts < MAX_CONTEXT_LIMIT_RECOVERY_ATTEMPTS
5020        {
5021            *transcript =
5022                crate::compaction::strip_last_message_before_error(std::mem::take(transcript));
5023        }
5024        Ok(true)
5025    }
5026
5027    async fn fail_turn_with_error(
5028        &self,
5029        thread_id: &ThreadId,
5030        turn_id: &TurnId,
5031        message: String,
5032    ) -> anyhow::Result<()> {
5033        self.persist_turn_item(
5034            thread_id,
5035            turn_id,
5036            &TranscriptItem::Error(ErrorRecord {
5037                message: message.clone(),
5038            }),
5039        )
5040        .await?;
5041        self.emit(RoderEvent::TurnFailed(TurnFailed {
5042            thread_id: thread_id.clone(),
5043            turn_id: turn_id.clone(),
5044            error: message.clone(),
5045            error_kind: None,
5046            usage: None,
5047            timestamp: OffsetDateTime::now_utc(),
5048        }))
5049        .await;
5050        self.complete_team_member_turn_with_result(
5051            thread_id,
5052            turn_id,
5053            TeamMemberStatus::Failed,
5054            None,
5055            Some(message),
5056        )
5057        .await?;
5058        Ok(())
5059    }
5060
5061    async fn fail_turn_due_to_deadline(
5062        &self,
5063        thread_id: &ThreadId,
5064        turn_id: &TurnId,
5065        deadline: OffsetDateTime,
5066        transcript: &[TranscriptItem],
5067    ) -> anyhow::Result<()> {
5068        let partial_result = turn_partial_result(transcript);
5069        self.emit(RoderEvent::TurnPartialResult(TurnPartialResult {
5070            thread_id: thread_id.clone(),
5071            turn_id: turn_id.clone(),
5072            summary: partial_result.clone(),
5073            timestamp: OffsetDateTime::now_utc(),
5074        }))
5075        .await;
5076        self.emit(RoderEvent::TurnDeadlineExceeded(TurnDeadlineExceeded {
5077            thread_id: thread_id.clone(),
5078            turn_id: turn_id.clone(),
5079            deadline,
5080            partial_result: partial_result.clone(),
5081            timestamp: OffsetDateTime::now_utc(),
5082        }))
5083        .await;
5084        let message = "turn deadline expired".to_string();
5085        self.persist_turn_item(
5086            thread_id,
5087            turn_id,
5088            &TranscriptItem::Error(ErrorRecord {
5089                message: format!("{message}: {partial_result}"),
5090            }),
5091        )
5092        .await?;
5093        self.emit(RoderEvent::TurnFailed(TurnFailed {
5094            thread_id: thread_id.clone(),
5095            turn_id: turn_id.clone(),
5096            error: message.clone(),
5097            error_kind: Some("deadline_timeout".to_string()),
5098            usage: None,
5099            timestamp: OffsetDateTime::now_utc(),
5100        }))
5101        .await;
5102        self.complete_team_member_turn_with_result(
5103            thread_id,
5104            turn_id,
5105            TeamMemberStatus::Failed,
5106            None,
5107            Some(format!("{message}: {partial_result}")),
5108        )
5109        .await?;
5110        Ok(())
5111    }
5112
5113    async fn start_deadline_finalization(
5114        &self,
5115        thread_id: &ThreadId,
5116        turn_id: &TurnId,
5117        transcript: &mut Vec<TranscriptItem>,
5118        remaining_seconds: u64,
5119    ) -> anyhow::Result<()> {
5120        let item = TranscriptItem::UserMessage(crate::deadline_policy::finalization_message(
5121            remaining_seconds,
5122        ));
5123        self.persist_turn_item(thread_id, turn_id, &item).await?;
5124        transcript.push(item);
5125        self.emit(RoderEvent::TurnPartialResult(TurnPartialResult {
5126            thread_id: thread_id.clone(),
5127            turn_id: turn_id.clone(),
5128            summary: turn_partial_result(transcript),
5129            timestamp: OffsetDateTime::now_utc(),
5130        }))
5131        .await;
5132        Ok(())
5133    }
5134
5135    /// Emits the reliability-limit event for a continuation (loop persistence)
5136    /// without failing the turn. The caller resets the failure counter and
5137    /// injects a nudge so the round loop keeps going.
5138    async fn record_reliability_limit_continuation(
5139        &self,
5140        thread_id: &ThreadId,
5141        turn_id: &TurnId,
5142        provider: &str,
5143        model: &str,
5144        limit: ReliabilityLimitHit,
5145    ) {
5146        self.emit(RoderEvent::ReliabilityLimitRecorded(
5147            ReliabilityLimitRecorded {
5148                context: ReliabilityContext {
5149                    thread_id: thread_id.clone(),
5150                    turn_id: turn_id.clone(),
5151                    tool_id: None,
5152                    tool_name: None,
5153                    provider: Some(provider.to_string()),
5154                    model: Some(model.to_string()),
5155                },
5156                error_class: limit.error_class,
5157                limit_kind: limit.limit_kind,
5158                decision: limit.decision,
5159                current: limit.current,
5160                limit: limit.limit,
5161                details: ReliabilityDetails::redacted(&limit.message),
5162                timestamp: OffsetDateTime::now_utc(),
5163            },
5164        ))
5165        .await;
5166    }
5167
5168    async fn fail_turn_due_to_reliability_limit(
5169        &self,
5170        thread_id: &ThreadId,
5171        turn_id: &TurnId,
5172        provider: &str,
5173        model: &str,
5174        limit: ReliabilityLimitHit,
5175        transcript: &[TranscriptItem],
5176    ) -> anyhow::Result<()> {
5177        self.emit(RoderEvent::ReliabilityLimitRecorded(
5178            ReliabilityLimitRecorded {
5179                context: ReliabilityContext {
5180                    thread_id: thread_id.clone(),
5181                    turn_id: turn_id.clone(),
5182                    tool_id: None,
5183                    tool_name: None,
5184                    provider: Some(provider.to_string()),
5185                    model: Some(model.to_string()),
5186                },
5187                error_class: limit.error_class,
5188                limit_kind: limit.limit_kind,
5189                decision: limit.decision,
5190                current: limit.current,
5191                limit: limit.limit,
5192                details: ReliabilityDetails::redacted(&limit.message),
5193                timestamp: OffsetDateTime::now_utc(),
5194            },
5195        ))
5196        .await;
5197        let partial_result = turn_partial_result(transcript);
5198        self.emit(RoderEvent::TurnPartialResult(TurnPartialResult {
5199            thread_id: thread_id.clone(),
5200            turn_id: turn_id.clone(),
5201            summary: partial_result.clone(),
5202            timestamp: OffsetDateTime::now_utc(),
5203        }))
5204        .await;
5205        let message = format!("reliability limit reached: {}", limit.message);
5206        self.persist_turn_item(
5207            thread_id,
5208            turn_id,
5209            &TranscriptItem::Error(ErrorRecord {
5210                message: format!("{message}: {partial_result}"),
5211            }),
5212        )
5213        .await?;
5214        self.emit(RoderEvent::TurnFailed(TurnFailed {
5215            thread_id: thread_id.clone(),
5216            turn_id: turn_id.clone(),
5217            error: message.clone(),
5218            error_kind: Some("reliability_limit".to_string()),
5219            usage: None,
5220            timestamp: OffsetDateTime::now_utc(),
5221        }))
5222        .await;
5223        self.complete_team_member_turn_with_result(
5224            thread_id,
5225            turn_id,
5226            TeamMemberStatus::Failed,
5227            None,
5228            Some(format!("{message}: {partial_result}")),
5229        )
5230        .await?;
5231        Ok(())
5232    }
5233
5234    async fn append_steers(
5235        &self,
5236        req: &StartTurnRequest,
5237        turn_id: &TurnId,
5238        transcript: &mut Vec<TranscriptItem>,
5239        steers: Vec<QueuedTurnSteer>,
5240    ) -> anyhow::Result<()> {
5241        for queued in steers {
5242            let mut steer = queued.message;
5243            steer.text = steer.text.trim().to_string();
5244            if steer.text.is_empty() && steer.images.is_empty() {
5245                continue;
5246            }
5247            let item = TranscriptItem::UserMessage(steer);
5248            self.persist_turn_item(&req.thread_id, turn_id, &item)
5249                .await?;
5250            transcript.push(item);
5251            if let Some(ack) = queued.mailbox_ack {
5252                self.teams
5253                    .mark_mailbox_messages_delivered(&ack.team_id, turn_id, &ack.message_ids)
5254                    .await?;
5255            }
5256        }
5257        Ok(())
5258    }
5259
5260    async fn persist_model_profile_segment(
5261        &self,
5262        thread_id: &ThreadId,
5263        turn_id: &TurnId,
5264        profile: Option<&ModelHarnessProfile>,
5265        provider: &str,
5266        model: &str,
5267        segment: &str,
5268    ) -> anyhow::Result<()> {
5269        let item = TranscriptItem::ProviderMetadata(model_profile_segment_metadata(
5270            profile, provider, model, segment,
5271        ));
5272        self.persist_turn_item(thread_id, turn_id, &item).await
5273    }
5274
5275    fn task_ledger_tool_specs(
5276        &self,
5277        profile: Option<&ModelHarnessProfile>,
5278    ) -> Vec<roder_api::tools::ToolSpec> {
5279        self.tool_registry
5280            .get(TASK_LEDGER_TOOL_NAME)
5281            .map(|tool| {
5282                tool.spec()
5283                    .normalized_for_model_profile(schema_policy_for_model(profile))
5284            })
5285            .into_iter()
5286            .collect()
5287    }
5288
5289    pub(crate) fn engine_for(&self, provider: &str) -> anyhow::Result<Arc<dyn InferenceEngine>> {
5290        self.registry
5291            .inference_engine(provider)
5292            .or_else(|| {
5293                self.registry
5294                    .default_inference_engine()
5295                    .filter(|engine| provider.is_empty() || engine.id() == provider)
5296            })
5297            .ok_or_else(|| anyhow::anyhow!("inference provider {provider:?} is not registered"))
5298    }
5299
5300    pub async fn emit(&self, event: RoderEvent) -> EventEnvelope {
5301        self.dispatch_local_hook_lifecycle(&event).await;
5302        if let Some(record) = Self::lifecycle_record_for_event(&event) {
5303            let _ = self.persist_turn_lifecycle_record(&record).await;
5304        }
5305        let envelope = self.bus.emit(event);
5306        if let (Some(store), Some(thread_id)) = (&self.thread_store, envelope.thread_id.as_ref())
5307            && should_persist_thread_event(thread_id)
5308        {
5309            let _ = store.append_event(thread_id, &envelope).await;
5310        }
5311        self.dispatch_event_sinks(&envelope).await;
5312        envelope
5313    }
5314
5315    async fn dispatch_local_hook_lifecycle(&self, event: &RoderEvent) {
5316        // SessionStart belongs to the session, not the turn: firing it from
5317        // TurnStarted re-ran setup hooks on every user message. ThreadCreated
5318        // and ThreadLoaded carry no turn id, so they borrow a synthetic one the
5319        // way the SessionEnd path does.
5320        let session_turn_id = SESSION_HOOK_TURN_ID.to_string();
5321        let (thread_id, turn_id, name, matcher, input) = match event {
5322            RoderEvent::ThreadCreated(event) => (
5323                &event.thread_id,
5324                &session_turn_id,
5325                "SessionStart",
5326                Some("startup"),
5327                serde_json::json!({"source":"startup"}),
5328            ),
5329            RoderEvent::ThreadLoaded(event) => (
5330                &event.thread_id,
5331                &session_turn_id,
5332                "SessionStart",
5333                Some("resume"),
5334                serde_json::json!({"source":"resume"}),
5335            ),
5336            RoderEvent::TurnCompleted(event) => (
5337                &event.thread_id,
5338                &event.turn_id,
5339                "Stop",
5340                None,
5341                serde_json::json!({"finishReason":event.finish_reason}),
5342            ),
5343            RoderEvent::TurnFailed(event) => (
5344                &event.thread_id,
5345                &event.turn_id,
5346                "Stop",
5347                None,
5348                serde_json::json!({"error":event.error}),
5349            ),
5350            RoderEvent::ContextCompactionStarted(event) => (
5351                &event.thread_id,
5352                &event.turn_id,
5353                "PreCompact",
5354                None,
5355                serde_json::json!({"originalItemCount":event.original_item_count,"originalEstimatedTokens":event.original_estimated_tokens}),
5356            ),
5357            RoderEvent::ContextCompactionRecorded(event) => (
5358                &event.thread_id,
5359                &event.turn_id,
5360                "PostCompact",
5361                None,
5362                serde_json::json!({"compactedItemCount":event.compacted_item_count,"compactedEstimatedTokens":event.compacted_estimated_tokens}),
5363            ),
5364            RoderEvent::SubagentStarted(event) => (
5365                &event.parent_thread_id,
5366                &event.parent_turn_id,
5367                "SubagentStart",
5368                Some(event.agent_type.as_str()),
5369                serde_json::json!({"subagentThreadId":event.thread_id,"agentType":event.agent_type,"description":event.description}),
5370            ),
5371            RoderEvent::SubagentCompleted(event) => (
5372                &event.parent_thread_id,
5373                &event.parent_turn_id,
5374                "SubagentStop",
5375                Some(event.agent_type.as_str()),
5376                serde_json::json!({"subagentThreadId":event.thread_id,"agentType":event.agent_type,"exitReason":event.exit_reason}),
5377            ),
5378            RoderEvent::SubagentFailed(event) => (
5379                &event.parent_thread_id,
5380                &event.parent_turn_id,
5381                "SubagentStop",
5382                Some(event.agent_type.as_str()),
5383                serde_json::json!({"subagentThreadId":event.thread_id,"agentType":event.agent_type,"error":event.error}),
5384            ),
5385            _ => return,
5386        };
5387        // One SessionStart per session, whichever event opened it.
5388        if name == "SessionStart" && !self.claim_session_hook_start(thread_id).await {
5389            return;
5390        }
5391        let workspace = self.config.read().await.workspace.clone();
5392        crate::hooks::run_lifecycle(
5393            self,
5394            thread_id,
5395            turn_id,
5396            workspace.as_deref(),
5397            name,
5398            matcher,
5399            input,
5400        )
5401        .await;
5402    }
5403
5404    /// Claim the single `SessionStart` dispatch for a thread, returning whether
5405    /// this caller won it.
5406    pub(crate) async fn claim_session_hook_start(&self, thread_id: &ThreadId) -> bool {
5407        self.session_hook_started
5408            .write()
5409            .await
5410            .insert(thread_id.clone())
5411    }
5412
5413    /// Records a projected thread item event and persists it through the configured thread store.
5414    ///
5415    /// App-server protocol notification bridges currently call this after translating runtime
5416    /// events into item events. Headless runtime consumers that subscribe to `RoderEvent` directly
5417    /// must make the same call if they need the derived item event stream persisted.
5418    pub async fn record_thread_item_event_kind(
5419        &self,
5420        thread_id: &ThreadId,
5421        turn_id: &TurnId,
5422        timestamp: OffsetDateTime,
5423        kind: ThreadItemEventKind,
5424    ) -> anyhow::Result<ThreadItemEvent> {
5425        let seq = self.next_thread_item_event_seq(thread_id).await?;
5426        let item_event = ThreadItemEvent {
5427            seq,
5428            event_id: uuid::Uuid::new_v4().to_string(),
5429            thread_id: thread_id.clone(),
5430            turn_id: turn_id.clone(),
5431            timestamp,
5432            event: kind,
5433        };
5434        if let Some(store) = &self.thread_store {
5435            store.append_item_event(thread_id, &item_event).await?;
5436        }
5437        self.remember_thread_item_event(&item_event).await?;
5438        Ok(item_event)
5439    }
5440
5441    async fn next_thread_item_event_seq(&self, thread_id: &ThreadId) -> anyhow::Result<u64> {
5442        self.ensure_thread_item_cache(thread_id).await?;
5443        Ok(self
5444            .thread_item_cache
5445            .lock()
5446            .await
5447            .next_item_event_seq(thread_id))
5448    }
5449
5450    pub async fn thread_item_exists(
5451        &self,
5452        thread_id: &ThreadId,
5453        turn_id: &TurnId,
5454        item_id: &str,
5455    ) -> anyhow::Result<bool> {
5456        self.ensure_thread_item_cache(thread_id).await?;
5457        Ok(self
5458            .thread_item_cache
5459            .lock()
5460            .await
5461            .thread_item_exists(thread_id, turn_id, item_id))
5462    }
5463
5464    pub async fn current_reasoning_item_id(
5465        &self,
5466        thread_id: &ThreadId,
5467        turn_id: &TurnId,
5468    ) -> anyhow::Result<Option<String>> {
5469        self.ensure_thread_item_cache(thread_id).await?;
5470        Ok(self
5471            .thread_item_cache
5472            .lock()
5473            .await
5474            .current_reasoning_item_id(thread_id, turn_id))
5475    }
5476
5477    async fn remember_thread_item_event(&self, item_event: &ThreadItemEvent) -> anyhow::Result<()> {
5478        self.ensure_thread_item_cache(&item_event.thread_id).await?;
5479        self.thread_item_cache
5480            .lock()
5481            .await
5482            .remember_item_event(item_event);
5483        Ok(())
5484    }
5485
5486    pub async fn latest_transcript_item_index(
5487        &self,
5488        thread_id: &ThreadId,
5489        turn_id: &TurnId,
5490    ) -> anyhow::Result<Option<usize>> {
5491        self.ensure_thread_item_cache(thread_id).await?;
5492        Ok(self
5493            .thread_item_cache
5494            .lock()
5495            .await
5496            .latest_transcript_item_index(thread_id, turn_id))
5497    }
5498
5499    async fn next_transcript_item_index(
5500        &self,
5501        thread_id: &ThreadId,
5502        turn_id: &TurnId,
5503    ) -> anyhow::Result<usize> {
5504        self.ensure_thread_item_cache(thread_id).await?;
5505        Ok(self
5506            .thread_item_cache
5507            .lock()
5508            .await
5509            .next_transcript_item_index(thread_id, turn_id))
5510    }
5511
5512    async fn remember_transcript_item_index(
5513        &self,
5514        thread_id: &ThreadId,
5515        turn_id: &TurnId,
5516        item_index: usize,
5517    ) -> anyhow::Result<()> {
5518        self.ensure_thread_item_cache(thread_id).await?;
5519        self.thread_item_cache
5520            .lock()
5521            .await
5522            .remember_transcript_item_index(thread_id, turn_id, item_index);
5523        Ok(())
5524    }
5525
5526    async fn ensure_thread_item_cache(&self, thread_id: &ThreadId) -> anyhow::Result<()> {
5527        if self
5528            .thread_item_cache
5529            .lock()
5530            .await
5531            .contains_thread(thread_id)
5532        {
5533            return Ok(());
5534        }
5535
5536        let snapshot = if let Some(store) = &self.thread_store {
5537            store.load_thread(thread_id).await?
5538        } else {
5539            None
5540        };
5541        self.thread_item_cache.lock().await.ensure_thread(
5542            thread_id,
5543            ThreadItemCacheEntry::from_snapshot(snapshot.as_ref()),
5544        );
5545        Ok(())
5546    }
5547
5548    pub(crate) async fn persist_turn_item(
5549        &self,
5550        thread_id: &ThreadId,
5551        turn_id: &TurnId,
5552        item: &TranscriptItem,
5553    ) -> anyhow::Result<()> {
5554        let item_index = self.next_transcript_item_index(thread_id, turn_id).await?;
5555        let timestamp = OffsetDateTime::now_utc();
5556        self.emit(RoderEvent::TranscriptItemAppended(TranscriptItemAppended {
5557            thread_id: thread_id.clone(),
5558            turn_id: turn_id.clone(),
5559            item_type: match item {
5560                TranscriptItem::UserMessage(_) => "user_message",
5561                TranscriptItem::AssistantMessage(_) => "assistant_message",
5562                TranscriptItem::ReasoningSummary(_) => "reasoning_summary",
5563                TranscriptItem::ToolCall(_) => "tool_call",
5564                TranscriptItem::ToolResult(_) => "tool_result",
5565                TranscriptItem::FileChange(_) => "file_change",
5566                TranscriptItem::ContextCompaction(_) => "context_compaction",
5567                TranscriptItem::Error(_) => "error",
5568                TranscriptItem::ProviderMetadata(_) => "provider_metadata",
5569            }
5570            .to_string(),
5571            item_index: Some(item_index),
5572            item: Some(item.clone()),
5573            timestamp,
5574        }))
5575        .await;
5576        self.remember_transcript_item_index(thread_id, turn_id, item_index)
5577            .await?;
5578        Ok(())
5579    }
5580}
5581
5582fn transcript_has_images(transcript: &[TranscriptItem]) -> bool {
5583    transcript.iter().any(|item| {
5584        matches!(
5585            item,
5586            TranscriptItem::UserMessage(message) if !message.images.is_empty()
5587        )
5588    })
5589}
5590
5591fn transcript_has_task_ledger(transcript: &[TranscriptItem]) -> bool {
5592    transcript.iter().any(|item| {
5593        matches!(
5594            item,
5595            TranscriptItem::ToolResult(result)
5596                if result.name.as_deref() == Some(TASK_LEDGER_TOOL_NAME) && !result.is_error
5597        )
5598    })
5599}
5600
5601fn task_ledger_completion_prompt(transcript: &[TranscriptItem]) -> Option<String> {
5602    let latest = transcript.iter().rev().find_map(|item| match item {
5603        TranscriptItem::ToolResult(result)
5604            if result.name.as_deref() == Some(TASK_LEDGER_TOOL_NAME) && !result.is_error =>
5605        {
5606            Some(result.result.as_str())
5607        }
5608        _ => None,
5609    })?;
5610    if !task_ledger_has_open_items(latest) {
5611        return None;
5612    }
5613
5614    let mut ledger = latest.chars().take(1500).collect::<String>();
5615    if latest.chars().nth(1500).is_some() {
5616        ledger.push_str("...");
5617    }
5618    Some(format!(
5619        "Task Ledger Completion Required: the latest task ledger still has pending or in-progress items. Do not provide a final answer yet. Use tools to complete the remaining scoreable work, create or update any required output files, then call `{TASK_LEDGER_TOOL_NAME}` with every task completed and evidence before finalizing.\n\nLatest ledger:\n{ledger}"
5620    ))
5621}
5622
5623fn task_ledger_deadline_completion_prompt(
5624    remaining_seconds: u64,
5625    reserve_seconds: u64,
5626    completion_prompt: &str,
5627) -> String {
5628    format!(
5629        "Eval deadline scoreable completion: {remaining_seconds} seconds remain in the {reserve_seconds}-second finalization reserve. Do not browse, search, or start slow work. Use the available tools now to create or update the required scoreable output files, run only a quick local check if needed, then update the task ledger to completed before finalizing.\n\n{completion_prompt}"
5630    )
5631}
5632
5633fn task_ledger_scoreable_checkpoint_prompt(
5634    remaining_seconds: u64,
5635    completion_prompt: &str,
5636) -> String {
5637    format!(
5638        "Scoreable Output Checkpoint: {remaining_seconds} seconds remain before the eval deadline. Before any further research, browsing, or long commands, use tools now to ensure the required output file(s) exist with the best evidence-backed answer, even if provisional. If a scoreable file already exists, read it and preserve that candidate unless you have stronger task-specific evidence for a replacement. Do not overwrite a plausible dated, historical, or local-evidence candidate with a current live-page, partial-coverage, or weaker guess merely to refresh the checkpoint. You may continue refining afterward, but do not apologize or finalize until the scoreable file exists and the task ledger is updated.\n\n{completion_prompt}"
5639    )
5640}
5641
5642fn task_ledger_has_open_items(ledger: &str) -> bool {
5643    ledger.lines().any(|line| {
5644        let line = line.trim_start();
5645        line.starts_with("- pending:") || line.starts_with("- in_progress:")
5646    })
5647}
5648
5649fn turn_deadline_for_config(cfg: &RuntimeConfig) -> Option<OffsetDateTime> {
5650    cfg.turn_deadline_seconds
5651        .filter(|seconds| *seconds > 0)
5652        .map(|seconds| OffsetDateTime::now_utc() + Duration::seconds(seconds as i64))
5653}
5654
5655fn deadline_expired(deadline: OffsetDateTime) -> bool {
5656    OffsetDateTime::now_utc() >= deadline
5657}
5658
5659pub(crate) fn deadline_remaining_seconds(deadline: Option<OffsetDateTime>) -> Option<u64> {
5660    let deadline = deadline?;
5661    if deadline <= OffsetDateTime::now_utc() {
5662        return Some(0);
5663    }
5664    Some(
5665        (deadline - OffsetDateTime::now_utc())
5666            .unsigned_abs()
5667            .as_secs()
5668            .max(1),
5669    )
5670}
5671
5672fn deadline_instant(deadline: OffsetDateTime) -> tokio::time::Instant {
5673    let now = OffsetDateTime::now_utc();
5674    if deadline <= now {
5675        return tokio::time::Instant::now();
5676    }
5677    tokio::time::Instant::now() + (deadline - now).unsigned_abs()
5678}
5679
5680fn inference_timeout_deadline(
5681    deadline: Option<OffsetDateTime>,
5682    runtime_profile: RuntimeProfile,
5683    task_ledger_required: bool,
5684    reserve_seconds: u64,
5685    finalization_requested: bool,
5686    task_ledger_scoreable_checkpoints: u8,
5687    transcript: &[TranscriptItem],
5688) -> Option<(OffsetDateTime, InferenceTimeoutAction)> {
5689    let deadline = deadline?;
5690    if runtime_profile == RuntimeProfile::Eval
5691        && task_ledger_required
5692        && !finalization_requested
5693        && task_ledger_scoreable_checkpoints < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
5694        && TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS > reserve_seconds
5695        && task_ledger_completion_prompt(transcript).is_some()
5696    {
5697        let checkpoint_deadline =
5698            deadline - Duration::seconds(TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS as i64);
5699        if checkpoint_deadline > OffsetDateTime::now_utc() {
5700            return Some((
5701                checkpoint_deadline,
5702                InferenceTimeoutAction::ScoreableCheckpoint,
5703            ));
5704        }
5705    }
5706    if runtime_profile == RuntimeProfile::Eval && !finalization_requested {
5707        return Some((
5708            deadline - Duration::seconds(reserve_seconds as i64),
5709            InferenceTimeoutAction::Finalization,
5710        ));
5711    }
5712    Some((deadline, InferenceTimeoutAction::Finalization))
5713}
5714
5715fn turn_partial_result(transcript: &[TranscriptItem]) -> String {
5716    let tool_results = transcript
5717        .iter()
5718        .filter(|item| matches!(item, TranscriptItem::ToolResult(_)))
5719        .count();
5720    let assistant_messages = transcript
5721        .iter()
5722        .filter(|item| matches!(item, TranscriptItem::AssistantMessage(_)))
5723        .count();
5724    format!(
5725        "partial turn state: {} transcript items, {assistant_messages} assistant messages, {tool_results} tool results",
5726        transcript.len()
5727    )
5728}
5729
5730fn reasoning_for_model(cfg: &RuntimeConfig, model: &str) -> ReasoningConfig {
5731    let level = effective_reasoning_for_model(cfg, model);
5732    match level.as_str() {
5733        "" | REASONING_NONE => ReasoningConfig::default(),
5734        level => ReasoningConfig {
5735            enabled: true,
5736            level: Some(level.to_string()),
5737        },
5738    }
5739}
5740
5741fn server_side_compaction_threshold(cfg: &RuntimeConfig, model: &str) -> Option<u32> {
5742    let entry = lookup_model(model)?;
5743    if !entry.supports_compaction {
5744        return None;
5745    }
5746    cfg.auto_compact_token_limit
5747        .or_else(|| {
5748            model_profile_for_model(cfg, model).and_then(|profile| profile.auto_compact_token_limit)
5749        })
5750        .or(Some(entry.auto_compact_token_limit))
5751        .filter(|threshold| *threshold > 0)
5752}
5753
5754pub(crate) fn tool_search_for_provider_model(
5755    cfg: &RuntimeConfig,
5756    provider: &str,
5757    model: &str,
5758) -> ToolSearchConfig {
5759    let mut resolved = cfg.tool_search.clone();
5760    if let Some(provider_config) = cfg.provider_tool_search.get(provider) {
5761        provider_config.apply_to(&mut resolved);
5762    }
5763    if let Some(model_config) = cfg.model_tool_search.get(model) {
5764        model_config.apply_to(&mut resolved);
5765    }
5766    resolved
5767}
5768
5769fn parallel_tool_calls_for_model(cfg: &RuntimeConfig, model: &str) -> bool {
5770    cfg.model_parallel_tool_calls
5771        .get(model)
5772        .copied()
5773        .or_else(|| {
5774            model_profile_for_model(cfg, model).and_then(|profile| profile.parallel_tool_calls)
5775        })
5776        .unwrap_or(true)
5777}
5778
5779/// Count the children an `agent_swarm` call will launch from its arguments
5780/// (item-based spawns plus resumes). Lenient: returns 0 on malformed input.
5781fn agent_swarm_child_count(arguments: &str) -> usize {
5782    serde_json::from_str::<roder_api::subagents::AgentSwarmRequest>(arguments)
5783        .map(|request| request.items.len() + request.resume_agent_ids.len())
5784        .unwrap_or(0)
5785}
5786
5787/// Parse `completed`/`failed`/`aborted` counts from an `<agent_swarm_result>`
5788/// summary line, e.g. `<summary>completed: 2, failed: 1</summary>`. Omitted
5789/// buckets default to 0. Returns `None` when the text is not a swarm result.
5790fn parse_swarm_counts(text: &str) -> Option<(usize, usize, usize)> {
5791    if !text.contains("<agent_swarm_result>") {
5792        return None;
5793    }
5794    let bucket = |label: &str| -> usize {
5795        let needle = format!("{label}: ");
5796        text.find(&needle)
5797            .map(|start| start + needle.len())
5798            .map(|start| {
5799                text[start..]
5800                    .chars()
5801                    .take_while(|c| c.is_ascii_digit())
5802                    .collect::<String>()
5803            })
5804            .and_then(|digits| digits.parse().ok())
5805            .unwrap_or(0)
5806    };
5807    Some((bucket("completed"), bucket("failed"), bucket("aborted")))
5808}
5809
5810fn effective_reasoning_for_model(cfg: &RuntimeConfig, model: &str) -> String {
5811    let base_reasoning = default_effective_reasoning_for_model(cfg, model);
5812    if cfg.dynamic_workflows.effort_profile == DynamicWorkflowEffortProfile::Ultracode {
5813        return ultracode_reasoning_level_for_model(
5814            model,
5815            &cfg.speed_policy.ultracode_reasoning,
5816            &base_reasoning,
5817        );
5818    }
5819    base_reasoning
5820}
5821
5822fn default_effective_reasoning_for_model(cfg: &RuntimeConfig, model: &str) -> String {
5823    let Some(entry) = lookup_model(model) else {
5824        return cfg
5825            .reasoning
5826            .clone()
5827            .unwrap_or_else(|| REASONING_NONE.to_string());
5828    };
5829    if entry.supported_reasoning.is_empty() {
5830        return REASONING_NONE.to_string();
5831    }
5832    cfg.reasoning
5833        .as_deref()
5834        .filter(|reasoning| {
5835            entry
5836                .supported_reasoning
5837                .iter()
5838                .any(|option| option.effort == *reasoning)
5839        })
5840        .map(str::to_string)
5841        .or_else(|| {
5842            model_profile_for_model(cfg, model)
5843                .and_then(|profile| profile.reasoning.orientation)
5844                .filter(|reasoning| {
5845                    entry
5846                        .supported_reasoning
5847                        .iter()
5848                        .any(|option| option.effort == reasoning)
5849                })
5850        })
5851        .unwrap_or_else(|| entry.default_reasoning.to_string())
5852}
5853
5854fn validate_reasoning_effort(model: &str, effort: &str) -> anyhow::Result<()> {
5855    if effort == REASONING_NONE && !model_supports_reasoning(model, effort) {
5856        return Ok(());
5857    }
5858    let Some(entry) = lookup_model(model) else {
5859        return Ok(());
5860    };
5861    if entry
5862        .supported_reasoning
5863        .iter()
5864        .any(|option| option.effort == effort)
5865    {
5866        Ok(())
5867    } else {
5868        anyhow::bail!("model {model} does not support reasoning effort {effort}")
5869    }
5870}
5871
5872fn validate_runtime_config_reasoning(cfg: &RuntimeConfig) -> anyhow::Result<()> {
5873    let Some(reasoning) = cfg.reasoning.as_deref() else {
5874        return Ok(());
5875    };
5876    let Some(entry) = lookup_model(&cfg.default_model) else {
5877        return Ok(());
5878    };
5879    if entry.provider != PROVIDER_GEMINI {
5880        return Ok(());
5881    }
5882    validate_reasoning_effort(&cfg.default_model, reasoning)
5883}
5884
5885fn validate_runtime_inference_router_config(
5886    registry: &ExtensionRegistry,
5887    cfg: &RuntimeConfig,
5888) -> anyhow::Result<()> {
5889    if !cfg.inference_router.enabled {
5890        return Ok(());
5891    }
5892    let Some(router_id) = cfg.inference_router.router_id.as_deref() else {
5893        anyhow::bail!("inference_router.enabled requires inference_router.router");
5894    };
5895    if registry.inference_router(router_id).is_some() {
5896        return Ok(());
5897    }
5898    let available = registry
5899        .inference_routers
5900        .iter()
5901        .map(|router| router.id())
5902        .collect::<Vec<_>>()
5903        .join(", ");
5904    if available.is_empty() {
5905        anyhow::bail!("inference router {router_id:?} is not registered");
5906    }
5907    anyhow::bail!(
5908        "inference router {router_id:?} is not registered; available routers: {available}"
5909    );
5910}
5911
5912fn model_supports_reasoning(model: &str, effort: &str) -> bool {
5913    lookup_model(model)
5914        .map(|entry| {
5915            entry
5916                .supported_reasoning
5917                .iter()
5918                .any(|option| option.effort == effort)
5919        })
5920        .unwrap_or(false)
5921}
5922
5923fn is_final_answer_phase(phase: Option<&str>) -> bool {
5924    phase.is_none_or(|phase| phase.is_empty() || phase == FINAL_ANSWER_PHASE)
5925}
5926
5927fn edit_tool_for_model<'a>(cfg: &'a RuntimeConfig, model: &'a str) -> Option<&'a str> {
5928    cfg.model_edit_tools
5929        .get(model)
5930        .map(String::as_str)
5931        .or_else(|| {
5932            cfg.model_profiles
5933                .get(model)
5934                .and_then(|profile| profile.edit_tool.as_deref())
5935        })
5936        .or_else(|| lookup_model(model).and_then(|entry| entry.edit_tool))
5937        .or(Some(EDIT_TOOL_EDIT))
5938}
5939
5940fn model_profile_for_model(cfg: &RuntimeConfig, model: &str) -> Option<ModelHarnessProfile> {
5941    cfg.model_profiles
5942        .get(model)
5943        .cloned()
5944        .or_else(|| built_in_model_profile(model))
5945}
5946
5947pub(crate) fn allowlist_permits(allowlist: &[String], tool_name: &str) -> bool {
5948    allowlist.is_empty() || allowlist.iter().any(|allowed| allowed == tool_name)
5949}
5950
5951/// Provider-aware profile resolution for the active turn.
5952///
5953/// Many model ids are shared across providers (for example Cursor proxies
5954/// `claude-opus-4-8`). Resolving the harness profile by id alone picks the
5955/// first catalog entry, which assigns cross-provider ids the wrong family and
5956/// instruction overlay (e.g. a `cursor/claude-opus-4-8` turn would otherwise
5957/// inherit the Anthropic overlay). Prefer the explicit `(provider, model)`
5958/// catalog entry, keeping user-configured profile overrides as the top
5959/// precedence.
5960fn model_profile_for_provider_model(
5961    cfg: &RuntimeConfig,
5962    provider: &str,
5963    model: &str,
5964) -> Option<ModelHarnessProfile> {
5965    cfg.model_profiles
5966        .get(model)
5967        .cloned()
5968        .or_else(|| built_in_model_profile_for_provider(provider, model))
5969}
5970
5971fn schema_policy_for_model(profile: Option<&ModelHarnessProfile>) -> ModelSchemaPolicy {
5972    profile
5973        .map(|profile| profile.schema_policy)
5974        .unwrap_or_default()
5975}
5976
5977fn model_profile_segment_metadata(
5978    profile: Option<&ModelHarnessProfile>,
5979    provider: &str,
5980    model: &str,
5981    segment: &str,
5982) -> serde_json::Value {
5983    serde_json::json!({
5984        "kind": MODEL_PROFILE_TRACE_KIND,
5985        "segment": segment,
5986        "provider": provider,
5987        "model": model,
5988        "profileModel": profile.map(|profile| profile.model.as_str()).unwrap_or(model),
5989        "providerFamily": profile.map(|profile| profile.provider_family),
5990        "editTool": profile.and_then(|profile| profile.edit_tool.as_deref()),
5991        "schemaPolicy": profile.map(|profile| profile.schema_policy),
5992        "instructionOverlay": profile.map(|profile| profile.instruction_overlay),
5993        "parallelToolCalls": profile.and_then(|profile| profile.parallel_tool_calls),
5994        "autoCompactTokenLimit": profile.and_then(|profile| profile.auto_compact_token_limit),
5995    })
5996}
5997
5998fn model_switch_summary(
5999    transcript: &[TranscriptItem],
6000    profile: Option<&ModelHarnessProfile>,
6001    provider: &str,
6002    model: &str,
6003    tools: &[roder_api::tools::ToolSpec],
6004) -> Option<String> {
6005    let previous = latest_model_profile_segment(transcript)?;
6006    let previous_model = previous
6007        .get("model")
6008        .and_then(serde_json::Value::as_str)
6009        .unwrap_or_default();
6010    let previous_provider = previous
6011        .get("provider")
6012        .and_then(serde_json::Value::as_str)
6013        .unwrap_or_default();
6014    if previous_model == model && previous_provider == provider {
6015        return None;
6016    }
6017
6018    let previous_profile = previous
6019        .get("profileModel")
6020        .and_then(serde_json::Value::as_str)
6021        .unwrap_or(previous_model);
6022    let current_profile = profile
6023        .map(|profile| profile.model.as_str())
6024        .unwrap_or(model);
6025    let previous_edit_tool = previous
6026        .get("editTool")
6027        .and_then(serde_json::Value::as_str)
6028        .unwrap_or("none");
6029    let current_edit_tool = profile
6030        .and_then(|profile| profile.edit_tool.as_deref())
6031        .unwrap_or("none");
6032    let tool_names = tools
6033        .iter()
6034        .map(|tool| tool.name.as_str())
6035        .take(12)
6036        .collect::<Vec<_>>()
6037        .join(", ");
6038    Some(format!(
6039        "{MODEL_SWITCH_SUMMARY_PREFIX} previous profile {previous_provider}/{previous_profile} used edit tool {previous_edit_tool}. Current profile {provider}/{current_profile} uses edit tool {current_edit_tool}. Available tools now: {}.",
6040        if tool_names.is_empty() {
6041            "none"
6042        } else {
6043            &tool_names
6044        }
6045    ))
6046}
6047
6048fn latest_model_profile_segment(transcript: &[TranscriptItem]) -> Option<&serde_json::Value> {
6049    transcript.iter().rev().find_map(|item| {
6050        let TranscriptItem::ProviderMetadata(value) = item else {
6051            return None;
6052        };
6053        (value.get("kind").and_then(serde_json::Value::as_str) == Some(MODEL_PROFILE_TRACE_KIND))
6054            .then_some(value)
6055    })
6056}
6057
6058pub fn validate_edit_tool(value: &str) -> anyhow::Result<()> {
6059    match value.trim() {
6060        EDIT_TOOL_PATCH | EDIT_TOOL_EDIT => Ok(()),
6061        _ => anyhow::bail!(
6062            "unsupported edit_tool {value:?}; allowed values: {EDIT_TOOL_PATCH}, {EDIT_TOOL_EDIT}"
6063        ),
6064    }
6065}
6066
6067fn should_persist_thread_event(thread_id: &str) -> bool {
6068    !is_synthetic_event_thread_id(thread_id)
6069}
6070
6071#[cfg(test)]
6072#[path = "runtime/item_identity_tests.rs"]
6073mod item_identity_tests;
6074
6075#[cfg(test)]
6076#[path = "runtime/codex_v2_tests.rs"]
6077mod codex_v2_tests;
6078
6079#[cfg(test)]
6080#[path = "runtime/codex_v2_lifecycle_tests.rs"]
6081mod codex_v2_lifecycle_tests;
6082
6083#[cfg(test)]
6084mod tests {
6085    use super::*;
6086    use futures::stream;
6087    use roder_api::catalog::{
6088        PROVIDER_MOCK, REASONING_HIGH, REASONING_LOW, REASONING_MEDIUM, REASONING_MINIMAL,
6089        REASONING_NONE, REASONING_XHIGH,
6090    };
6091    use roder_api::extension::ExtensionRegistryBuilder;
6092    use roder_api::inference::{
6093        CompletionMetadata, InferenceCapabilities, InferenceEngine, InferenceEventStream,
6094        InferenceProviderContext, InferenceTurnContext, MessageDelta, ModelDescriptor,
6095        ModelInstructionOverlay, ModelProfileReasoning, ModelSchemaPolicy, ProviderFamily,
6096        ReasoningEffortDescriptor,
6097    };
6098    use roder_api::inference_routing::{
6099        InferenceRouter, InferenceRoutingContext, InferenceRoutingDecision, InferenceRoutingOutcome,
6100    };
6101    use roder_api::thread::ThreadStoreFactory;
6102    use roder_api::tools::{ToolContributor, ToolExecutor, ToolSpec};
6103    use roder_ext_jsonl_thread_store::store::JsonlThreadStoreFactory;
6104    use std::sync::Mutex as StdMutex;
6105
6106    fn test_workspace() -> String {
6107        std::env::current_dir().unwrap().display().to_string()
6108    }
6109
6110    #[test]
6111    fn interactive_explicit_turn_deadline_has_bounded_remaining_time() {
6112        let config = RuntimeConfig {
6113            runtime_profile: RuntimeProfile::Interactive,
6114            turn_deadline_seconds: Some(60),
6115            ..RuntimeConfig::default()
6116        };
6117
6118        let deadline = turn_deadline_for_config(&config).expect("configured turn deadline");
6119        let remaining = deadline_remaining_seconds(Some(deadline));
6120
6121        assert!(
6122            matches!(remaining, Some(1..=60)),
6123            "interactive deadline must be positive and no longer than configured: {remaining:?}"
6124        );
6125    }
6126
6127    #[test]
6128    fn interactive_without_turn_deadline_stays_unbounded() {
6129        let config = RuntimeConfig {
6130            runtime_profile: RuntimeProfile::Interactive,
6131            turn_deadline_seconds: None,
6132            ..RuntimeConfig::default()
6133        };
6134
6135        assert_eq!(turn_deadline_for_config(&config), None);
6136    }
6137
6138    struct MetadataMissingStore;
6139
6140    #[async_trait::async_trait]
6141    impl ThreadStore for MetadataMissingStore {
6142        fn id(&self) -> roder_api::thread::ThreadStoreId {
6143            "metadata-missing-store".to_string()
6144        }
6145
6146        async fn create_thread(&self, metadata: ThreadMetadata) -> anyhow::Result<ThreadMetadata> {
6147            Ok(metadata)
6148        }
6149
6150        async fn list_threads(&self) -> anyhow::Result<Vec<ThreadMetadata>> {
6151            Ok(Vec::new())
6152        }
6153
6154        async fn load_thread(
6155            &self,
6156            _thread_id: &ThreadId,
6157        ) -> anyhow::Result<Option<ThreadSnapshot>> {
6158            Ok(Some(ThreadSnapshot {
6159                metadata: None,
6160                ..ThreadSnapshot::default()
6161            }))
6162        }
6163
6164        async fn append_event(
6165            &self,
6166            _thread_id: &ThreadId,
6167            _envelope: &EventEnvelope,
6168        ) -> anyhow::Result<()> {
6169            Ok(())
6170        }
6171    }
6172
6173    struct MetadataMissingStoreFactory;
6174
6175    impl ThreadStoreFactory for MetadataMissingStoreFactory {
6176        fn id(&self) -> roder_api::thread::ThreadStoreId {
6177            "metadata-missing-store".to_string()
6178        }
6179
6180        fn create(&self) -> Arc<dyn ThreadStore> {
6181            Arc::new(MetadataMissingStore)
6182        }
6183    }
6184
6185    #[test]
6186    fn synthetic_app_server_events_are_not_thread_events() {
6187        for thread_id in ["app-server", "runtime", "thread-workflow"] {
6188            assert!(!should_persist_thread_event(thread_id));
6189        }
6190        assert!(should_persist_thread_event("thread-discovery"));
6191        assert!(should_persist_thread_event("thread-plan"));
6192        assert!(should_persist_thread_event("thread-process"));
6193        assert!(should_persist_thread_event("thread-1"));
6194    }
6195
6196    #[test]
6197    fn server_side_compaction_uses_catalog_ninety_percent_default() {
6198        assert_eq!(
6199            server_side_compaction_threshold(&RuntimeConfig::default(), "gpt-5.5"),
6200            Some(945_000)
6201        );
6202        assert_eq!(
6203            server_side_compaction_threshold(&RuntimeConfig::default(), "gpt-5.3-codex-spark"),
6204            Some(115_200)
6205        );
6206    }
6207
6208    #[test]
6209    fn server_side_compaction_respects_explicit_config_override() {
6210        let cfg = RuntimeConfig {
6211            auto_compact_token_limit: Some(123_456),
6212            ..RuntimeConfig::default()
6213        };
6214
6215        assert_eq!(
6216            server_side_compaction_threshold(&cfg, "gpt-5.5"),
6217            Some(123_456)
6218        );
6219    }
6220
6221    #[tokio::test]
6222    async fn pre_request_compaction_runs_when_server_side_model_is_at_context_window() {
6223        let captured = Arc::new(StdMutex::new(None));
6224        let mut builder = ExtensionRegistryBuilder::new();
6225        builder.inference_engine(Arc::new(CapturingEngine {
6226            request: captured.clone(),
6227        }));
6228        let thread_root = std::env::temp_dir().join(format!(
6229            "roder-pre-request-compaction-{}",
6230            uuid::Uuid::new_v4()
6231        ));
6232        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
6233            base_path: thread_root.clone(),
6234        }));
6235        let runtime = Arc::new(
6236            Runtime::new(
6237                builder.build().unwrap(),
6238                RuntimeConfig {
6239                    default_provider: PROVIDER_MOCK.to_string(),
6240                    default_model: "gpt-5.5".to_string(),
6241                    file_backed_dynamic_context: true,
6242                    ..RuntimeConfig::default()
6243                },
6244            )
6245            .unwrap(),
6246        );
6247        let thread_id = runtime
6248            .create_thread(Some("Pre-request compaction".to_string()))
6249            .await
6250            .unwrap()
6251            .thread_id;
6252        let old_turn = "old-turn".to_string();
6253        runtime
6254            .persist_turn_item(
6255                &thread_id,
6256                &old_turn,
6257                &TranscriptItem::UserMessage(UserMessage::text("old context ".repeat(4_300_000))),
6258            )
6259            .await
6260            .unwrap();
6261
6262        let mut events = runtime.subscribe_events();
6263        runtime
6264            .start_turn(StartTurnRequest {
6265                thread_id: thread_id.clone(),
6266                message: "continue".to_string(),
6267                images: Vec::new(),
6268                provider_override: None,
6269                model_override: None,
6270                reasoning_override: None,
6271                workspace: test_workspace(),
6272                instructions: InstructionBundle::default(),
6273                developer_context: None,
6274                task_ledger_required: false,
6275                service_tier_override: None,
6276            })
6277            .await
6278            .unwrap();
6279        loop {
6280            let envelope = tokio::time::timeout(std::time::Duration::from_secs(5), events.recv())
6281                .await
6282                .unwrap()
6283                .unwrap();
6284            if envelope.thread_id.as_deref() == Some(&thread_id)
6285                && matches!(envelope.event, RoderEvent::TurnCompleted(_))
6286            {
6287                break;
6288            }
6289        }
6290
6291        let request = captured.lock().unwrap().clone().unwrap();
6292        assert!(
6293            matches!(
6294                request.transcript.first(),
6295                Some(TranscriptItem::ContextCompaction(_))
6296            ),
6297            "provider request should start with a local emergency compaction item"
6298        );
6299        assert!(
6300            request.transcript.len() < 4,
6301            "provider request should not replay the full oversized prior transcript: {:?}",
6302            request.transcript
6303        );
6304
6305        let _ = std::fs::remove_dir_all(thread_root);
6306    }
6307
6308    #[tokio::test]
6309    async fn continue_after_context_window_failure_compacts_before_provider_request() {
6310        let captured = Arc::new(StdMutex::new(None));
6311        let mut builder = ExtensionRegistryBuilder::new();
6312        builder.inference_engine(Arc::new(CapturingEngine {
6313            request: captured.clone(),
6314        }));
6315        let thread_root = std::env::temp_dir().join(format!(
6316            "roder-context-failure-continue-{}",
6317            uuid::Uuid::new_v4()
6318        ));
6319        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
6320            base_path: thread_root.clone(),
6321        }));
6322        let runtime = Arc::new(
6323            Runtime::new(
6324                builder.build().unwrap(),
6325                RuntimeConfig {
6326                    default_provider: PROVIDER_MOCK.to_string(),
6327                    default_model: "gpt-5.5".to_string(),
6328                    file_backed_dynamic_context: true,
6329                    ..RuntimeConfig::default()
6330                },
6331            )
6332            .unwrap(),
6333        );
6334        let thread_id = runtime
6335            .create_thread(Some("Context failure continue".to_string()))
6336            .await
6337            .unwrap()
6338            .thread_id;
6339        let failed_turn = "failed-turn".to_string();
6340        runtime
6341            .persist_turn_item(
6342                &thread_id,
6343                &failed_turn,
6344                &TranscriptItem::UserMessage(UserMessage::text("old work ".repeat(10_000))),
6345            )
6346            .await
6347            .unwrap();
6348        runtime
6349            .persist_turn_item(
6350                &thread_id,
6351                &failed_turn,
6352                &TranscriptItem::Error(ErrorRecord {
6353                    message: "Your input exceeds the context window of this model. Please adjust your input and try again."
6354                        .to_string(),
6355                }),
6356            )
6357            .await
6358            .unwrap();
6359
6360        let mut events = runtime.subscribe_events();
6361        runtime
6362            .start_turn(StartTurnRequest {
6363                thread_id: thread_id.clone(),
6364                message: "continue".to_string(),
6365                images: Vec::new(),
6366                provider_override: None,
6367                model_override: None,
6368                reasoning_override: None,
6369                workspace: test_workspace(),
6370                instructions: InstructionBundle::default(),
6371                developer_context: None,
6372                task_ledger_required: false,
6373                service_tier_override: None,
6374            })
6375            .await
6376            .unwrap();
6377        loop {
6378            let envelope = tokio::time::timeout(std::time::Duration::from_secs(5), events.recv())
6379                .await
6380                .unwrap()
6381                .unwrap();
6382            if envelope.thread_id.as_deref() == Some(&thread_id)
6383                && matches!(envelope.event, RoderEvent::TurnCompleted(_))
6384            {
6385                break;
6386            }
6387        }
6388
6389        let request = captured.lock().unwrap().clone().unwrap();
6390        assert!(
6391            matches!(
6392                request.transcript.first(),
6393                Some(TranscriptItem::ContextCompaction(_))
6394            ),
6395            "provider request after context-window failure should start with local compaction"
6396        );
6397        assert!(
6398            request
6399                .transcript
6400                .iter()
6401                .any(|item| matches!(item, TranscriptItem::UserMessage(message) if message.text == "continue")),
6402            "current continue prompt must be preserved: {:?}",
6403            request.transcript
6404        );
6405        assert!(
6406            !request.transcript.iter().any(
6407                |item| matches!(item, TranscriptItem::Error(error) if error.message.contains("context window"))
6408            ),
6409            "raw prior context-window error should be summarized, not replayed: {:?}",
6410            request.transcript
6411        );
6412
6413        let _ = std::fs::remove_dir_all(thread_root);
6414    }
6415
6416    #[tokio::test]
6417    async fn workspace_for_thread_falls_back_when_metadata_is_missing() {
6418        let workspace = test_workspace();
6419        let mut builder = ExtensionRegistryBuilder::new();
6420        builder.inference_engine(Arc::new(FakeInferenceEngine));
6421        builder.thread_store_factory(Arc::new(MetadataMissingStoreFactory));
6422        let runtime = Runtime::new(
6423            builder.build().unwrap(),
6424            RuntimeConfig {
6425                workspace: Some(workspace.clone()),
6426                ..RuntimeConfig::default()
6427            },
6428        )
6429        .unwrap();
6430
6431        let resolved = runtime
6432            .workspace_for_thread(&ThreadId::from("thread-workflow"))
6433            .await
6434            .unwrap();
6435
6436        assert_eq!(resolved, workspace);
6437    }
6438
6439    #[tokio::test]
6440    async fn automations_can_create_project_thread_with_model_overrides() {
6441        let runtime = Runtime::fake().unwrap();
6442        let workspace = std::env::temp_dir().join("project");
6443        let metadata = runtime
6444            .create_thread_with(CreateThreadRequest {
6445                title: Some("Automation: nightly status".to_string()),
6446                workspace: workspace.display().to_string(),
6447                workspace_id: None,
6448                root_id: None,
6449                provider: Some("mock".to_string()),
6450                model: Some("mock".to_string()),
6451                selection_mode: None,
6452                tool_allowlist: Vec::new(),
6453                developer_instructions: None,
6454                external_tools: Vec::new(),
6455                runner: None,
6456            })
6457            .await
6458            .unwrap();
6459
6460        assert_eq!(
6461            metadata.title.as_deref(),
6462            Some("Automation: nightly status")
6463        );
6464        assert_eq!(metadata.workspace, workspace.display().to_string());
6465        assert_eq!(metadata.provider.as_deref(), Some("mock"));
6466        assert_eq!(metadata.model.as_deref(), Some("mock"));
6467    }
6468
6469    #[tokio::test]
6470    async fn prior_provider_compaction_boundary_is_used_on_next_turn() {
6471        let captured = Arc::new(StdMutex::new(None));
6472        let mut builder = ExtensionRegistryBuilder::new();
6473        builder.inference_engine(Arc::new(CapturingEngine {
6474            request: captured.clone(),
6475        }));
6476        let thread_root = std::env::temp_dir().join(format!(
6477            "roder-provider-compaction-boundary-{}",
6478            uuid::Uuid::new_v4()
6479        ));
6480        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
6481            base_path: thread_root.clone(),
6482        }));
6483        let runtime = Arc::new(
6484            Runtime::new(
6485                builder.build().unwrap(),
6486                RuntimeConfig {
6487                    default_provider: PROVIDER_MOCK.to_string(),
6488                    default_model: "gpt-5.5".to_string(),
6489                    ..RuntimeConfig::default()
6490                },
6491            )
6492            .unwrap(),
6493        );
6494        let thread_id = runtime
6495            .create_thread(Some("Provider compaction boundary".to_string()))
6496            .await
6497            .unwrap()
6498            .thread_id;
6499        let old_turn = "old-turn".to_string();
6500        runtime
6501            .persist_turn_item(
6502                &thread_id,
6503                &old_turn,
6504                &TranscriptItem::UserMessage(UserMessage::text(
6505                    "old history that must not be replayed after provider compaction",
6506                )),
6507            )
6508            .await
6509            .unwrap();
6510        runtime
6511            .persist_turn_item(
6512                &thread_id,
6513                &old_turn,
6514                &TranscriptItem::AssistantMessage(AssistantMessage {
6515                    text: "old answer".to_string(),
6516                    phase: None,
6517                }),
6518            )
6519            .await
6520            .unwrap();
6521        runtime
6522            .persist_turn_item(
6523                &thread_id,
6524                &old_turn,
6525                &TranscriptItem::ProviderMetadata(serde_json::json!({
6526                    "output": [{
6527                        "id": "cmp_1",
6528                        "type": "compaction",
6529                        "encrypted_content": "opaque-state"
6530                    }]
6531                })),
6532            )
6533            .await
6534            .unwrap();
6535        runtime
6536            .persist_turn_item(
6537                &thread_id,
6538                &old_turn,
6539                &TranscriptItem::AssistantMessage(AssistantMessage {
6540                    text: "after compact".to_string(),
6541                    phase: None,
6542                }),
6543            )
6544            .await
6545            .unwrap();
6546
6547        let mut events = runtime.subscribe_events();
6548        runtime
6549            .start_turn(StartTurnRequest {
6550                thread_id: thread_id.clone(),
6551                message: "continue".to_string(),
6552                images: Vec::new(),
6553                provider_override: None,
6554                model_override: None,
6555                reasoning_override: None,
6556                workspace: test_workspace(),
6557                instructions: InstructionBundle::default(),
6558                developer_context: None,
6559                task_ledger_required: false,
6560                service_tier_override: None,
6561            })
6562            .await
6563            .unwrap();
6564        loop {
6565            let envelope = tokio::time::timeout(std::time::Duration::from_secs(5), events.recv())
6566                .await
6567                .unwrap()
6568                .unwrap();
6569            if envelope.thread_id.as_deref() == Some(&thread_id)
6570                && matches!(envelope.event, RoderEvent::TurnCompleted(_))
6571            {
6572                break;
6573            }
6574        }
6575
6576        let request = captured.lock().unwrap().clone().unwrap();
6577        let compaction_idx = request.transcript.iter().position(|item| {
6578            matches!(
6579                item,
6580                TranscriptItem::ProviderMetadata(metadata)
6581                    if crate::compaction::provider_metadata_has_compaction(metadata)
6582            )
6583        });
6584        assert!(
6585            compaction_idx.is_some(),
6586            "next provider request should include the provider compaction item: {:?}",
6587            request.transcript
6588        );
6589        // Skill/context injectors may prepend non-history items, but no prior-turn
6590        // conversation may appear before the latest provider compaction boundary.
6591        let history_before_boundary =
6592            request
6593                .transcript
6594                .iter()
6595                .take(compaction_idx.unwrap())
6596                .any(|item| match item {
6597                    TranscriptItem::AssistantMessage(_)
6598                    | TranscriptItem::ToolCall(_)
6599                    | TranscriptItem::ToolResult(_) => true,
6600                    TranscriptItem::UserMessage(message) => {
6601                        !message.text.contains("<skills>") && message.text != "continue"
6602                    }
6603                    _ => false,
6604                });
6605        assert!(
6606            !history_before_boundary,
6607            "pre-provider-compaction conversation must not be replayed: {:?}",
6608            request.transcript
6609        );
6610        assert!(
6611            !request.transcript.iter().any(|item| {
6612                matches!(
6613                    item,
6614                    TranscriptItem::UserMessage(message)
6615                        if message.text.contains("old history that must not be replayed")
6616                )
6617            }),
6618            "pre-provider-compaction history must not be replayed: {:?}",
6619            request.transcript
6620        );
6621        assert!(
6622            request.transcript.iter().any(|item| {
6623                matches!(
6624                    item,
6625                    TranscriptItem::UserMessage(message) if message.text == "continue"
6626                )
6627            }),
6628            "current continue prompt must be preserved: {:?}",
6629            request.transcript
6630        );
6631        assert_eq!(
6632            request.runtime.auto_compact_token_limit,
6633            Some(945_000),
6634            "gpt-5.5 should keep server-side auto compaction configured"
6635        );
6636
6637        let _ = std::fs::remove_dir_all(thread_root);
6638    }
6639
6640    #[test]
6641    fn server_side_compaction_is_only_enabled_for_supported_models() {
6642        let cfg = RuntimeConfig {
6643            auto_compact_token_limit: Some(123_456),
6644            ..RuntimeConfig::default()
6645        };
6646
6647        assert_eq!(server_side_compaction_threshold(&cfg, "mock"), None);
6648        assert_eq!(
6649            server_side_compaction_threshold(&cfg, "codex-auto-review"),
6650            None
6651        );
6652    }
6653
6654    #[test]
6655    fn reasoning_is_disabled_for_models_without_reasoning_support() {
6656        let cfg = RuntimeConfig {
6657            reasoning: Some(REASONING_HIGH.to_string()),
6658            ..RuntimeConfig::default()
6659        };
6660
6661        assert_eq!(
6662            effective_reasoning_for_model(&cfg, "claude-haiku-4-5-20251001"),
6663            REASONING_NONE
6664        );
6665        assert_eq!(
6666            reasoning_for_model(&cfg, "claude-haiku-4-5-20251001"),
6667            ReasoningConfig::default()
6668        );
6669    }
6670
6671    #[test]
6672    fn unsupported_configured_reasoning_falls_back_to_model_default() {
6673        let cfg = RuntimeConfig {
6674            reasoning: Some(REASONING_MINIMAL.to_string()),
6675            ..RuntimeConfig::default()
6676        };
6677
6678        assert_eq!(
6679            effective_reasoning_for_model(&cfg, "gpt-5.5"),
6680            REASONING_MEDIUM
6681        );
6682    }
6683
6684    #[test]
6685    fn unsupported_configured_gemini_reasoning_is_rejected() {
6686        let mut builder = ExtensionRegistryBuilder::new();
6687        builder.inference_engine(std::sync::Arc::new(FakeInferenceEngine));
6688
6689        let err = match Runtime::new(
6690            builder.build().unwrap(),
6691            RuntimeConfig {
6692                default_model: "gemini-3.5-flash".to_string(),
6693                reasoning: Some(REASONING_XHIGH.to_string()),
6694                ..RuntimeConfig::default()
6695            },
6696        ) {
6697            Ok(_) => panic!("expected unsupported Gemini reasoning to be rejected"),
6698            Err(err) => err,
6699        };
6700
6701        assert!(
6702            err.to_string()
6703                .contains("model gemini-3.5-flash does not support reasoning effort xhigh")
6704        );
6705    }
6706
6707    #[tokio::test]
6708    async fn selecting_none_for_non_reasoning_model_preserves_stored_preference() {
6709        let runtime = Runtime::new(
6710            Runtime::fake().unwrap().registry,
6711            RuntimeConfig {
6712                reasoning: Some(REASONING_HIGH.to_string()),
6713                ..RuntimeConfig::default()
6714            },
6715        )
6716        .unwrap();
6717
6718        let cfg = runtime
6719            .select_provider(
6720                roder_api::catalog::PROVIDER_MOCK.to_string(),
6721                Some("claude-haiku-4-5-20251001".to_string()),
6722                Some(REASONING_NONE.to_string()),
6723            )
6724            .await
6725            .unwrap();
6726
6727        assert_eq!(cfg.reasoning.as_deref(), Some(REASONING_HIGH));
6728        assert_eq!(runtime.effective_reasoning().await, REASONING_NONE);
6729    }
6730
6731    #[tokio::test]
6732    async fn selecting_none_for_model_that_supports_none_updates_preference() {
6733        let runtime = Runtime::new(
6734            Runtime::fake().unwrap().registry,
6735            RuntimeConfig {
6736                reasoning: Some(REASONING_HIGH.to_string()),
6737                ..RuntimeConfig::default()
6738            },
6739        )
6740        .unwrap();
6741
6742        let cfg = runtime
6743            .select_provider(
6744                roder_api::catalog::PROVIDER_MOCK.to_string(),
6745                Some("mock".to_string()),
6746                Some(REASONING_NONE.to_string()),
6747            )
6748            .await
6749            .unwrap();
6750
6751        assert_eq!(cfg.reasoning.as_deref(), Some(REASONING_NONE));
6752    }
6753
6754    #[test]
6755    fn parallel_tool_calls_default_on_with_model_override() {
6756        assert!(parallel_tool_calls_for_model(
6757            &RuntimeConfig::default(),
6758            "custom-model"
6759        ));
6760
6761        let cfg = RuntimeConfig {
6762            model_parallel_tool_calls: std::collections::HashMap::from([(
6763                "custom-model".to_string(),
6764                false,
6765            )]),
6766            ..RuntimeConfig::default()
6767        };
6768
6769        assert!(!parallel_tool_calls_for_model(&cfg, "custom-model"));
6770        assert!(parallel_tool_calls_for_model(&cfg, "other-model"));
6771    }
6772
6773    #[test]
6774    fn profile_parallel_tool_calls_applies_between_config_and_default() {
6775        let cfg = RuntimeConfig {
6776            model_profiles: std::collections::HashMap::from([(
6777                "gpt-5.5".to_string(),
6778                test_model_profile("gpt-5.5"),
6779            )]),
6780            ..RuntimeConfig::default()
6781        };
6782
6783        assert!(!parallel_tool_calls_for_model(&cfg, "gpt-5.5"));
6784
6785        let cfg = RuntimeConfig {
6786            model_parallel_tool_calls: std::collections::HashMap::from([(
6787                "gpt-5.5".to_string(),
6788                true,
6789            )]),
6790            ..cfg
6791        };
6792
6793        assert!(parallel_tool_calls_for_model(&cfg, "gpt-5.5"));
6794    }
6795
6796    struct CapturingEngine {
6797        request: Arc<StdMutex<Option<AgentInferenceRequest>>>,
6798    }
6799
6800    #[async_trait::async_trait]
6801    impl InferenceEngine for CapturingEngine {
6802        fn id(&self) -> String {
6803            roder_api::catalog::PROVIDER_MOCK.to_string()
6804        }
6805
6806        fn capabilities(&self) -> InferenceCapabilities {
6807            InferenceCapabilities::coding_agent_default()
6808        }
6809
6810        async fn list_models(
6811            &self,
6812            _ctx: InferenceProviderContext<'_>,
6813        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
6814            Ok(roder_api::catalog::models_for_provider(
6815                roder_api::catalog::PROVIDER_MOCK,
6816                true,
6817            ))
6818        }
6819
6820        async fn stream_turn(
6821            &self,
6822            _ctx: InferenceTurnContext<'_>,
6823            request: AgentInferenceRequest,
6824        ) -> anyhow::Result<InferenceEventStream> {
6825            *self.request.lock().unwrap() = Some(request);
6826            Ok(Box::pin(stream::iter(vec![
6827                Ok(InferenceEvent::MessageDelta(MessageDelta {
6828                    text: "done".to_string(),
6829                    phase: None,
6830                })),
6831                Ok(InferenceEvent::Completed(CompletionMetadata {
6832                    stop_reason: Some("stop".to_string()),
6833                    provider_response_id: None,
6834                })),
6835            ])))
6836        }
6837    }
6838
6839    struct RoutingCaptureEngine {
6840        id: &'static str,
6841        models: Vec<ModelDescriptor>,
6842        requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
6843    }
6844
6845    #[async_trait::async_trait]
6846    impl InferenceEngine for RoutingCaptureEngine {
6847        fn id(&self) -> String {
6848            self.id.to_string()
6849        }
6850
6851        fn capabilities(&self) -> InferenceCapabilities {
6852            InferenceCapabilities::coding_agent_default()
6853        }
6854
6855        async fn list_models(
6856            &self,
6857            _ctx: InferenceProviderContext<'_>,
6858        ) -> anyhow::Result<Vec<ModelDescriptor>> {
6859            Ok(self.models.clone())
6860        }
6861
6862        async fn stream_turn(
6863            &self,
6864            _ctx: InferenceTurnContext<'_>,
6865            request: AgentInferenceRequest,
6866        ) -> anyhow::Result<InferenceEventStream> {
6867            self.requests.lock().unwrap().push(request);
6868            Ok(Box::pin(stream::iter(vec![
6869                Ok(InferenceEvent::MessageDelta(MessageDelta {
6870                    text: "routed".to_string(),
6871                    phase: None,
6872                })),
6873                Ok(InferenceEvent::Completed(CompletionMetadata {
6874                    stop_reason: Some("stop".to_string()),
6875                    provider_response_id: None,
6876                })),
6877            ])))
6878        }
6879    }
6880
6881    struct StaticRouter {
6882        id: &'static str,
6883        decision: InferenceRoutingDecision,
6884        contexts: Arc<StdMutex<Vec<InferenceRoutingContext>>>,
6885    }
6886
6887    #[async_trait::async_trait]
6888    impl InferenceRouter for StaticRouter {
6889        fn id(&self) -> String {
6890            self.id.to_string()
6891        }
6892
6893        async fn route(
6894            &self,
6895            context: InferenceRoutingContext,
6896        ) -> anyhow::Result<InferenceRoutingDecision> {
6897            self.contexts.lock().unwrap().push(context);
6898            Ok(self.decision.clone())
6899        }
6900    }
6901
6902    fn routing_test_model(id: &str, supported_reasoning: &[&str]) -> ModelDescriptor {
6903        ModelDescriptor {
6904            id: id.to_string(),
6905            name: id.to_string(),
6906            context_window: Some(128_000),
6907            default_reasoning: supported_reasoning
6908                .first()
6909                .map(|effort| (*effort).to_string()),
6910            supported_reasoning: supported_reasoning
6911                .iter()
6912                .map(|effort| ReasoningEffortDescriptor {
6913                    effort: (*effort).to_string(),
6914                    description: format!("{effort} reasoning"),
6915                })
6916                .collect(),
6917        }
6918    }
6919
6920    struct TaskLedgerCompletionGateEngine {
6921        calls: StdMutex<u32>,
6922        requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
6923    }
6924
6925    #[async_trait::async_trait]
6926    impl InferenceEngine for TaskLedgerCompletionGateEngine {
6927        fn id(&self) -> String {
6928            roder_api::catalog::PROVIDER_MOCK.to_string()
6929        }
6930
6931        fn capabilities(&self) -> InferenceCapabilities {
6932            InferenceCapabilities::coding_agent_default()
6933        }
6934
6935        async fn list_models(
6936            &self,
6937            _ctx: InferenceProviderContext<'_>,
6938        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
6939            Ok(roder_api::catalog::models_for_provider(
6940                roder_api::catalog::PROVIDER_MOCK,
6941                true,
6942            ))
6943        }
6944
6945        async fn stream_turn(
6946            &self,
6947            _ctx: InferenceTurnContext<'_>,
6948            request: AgentInferenceRequest,
6949        ) -> anyhow::Result<InferenceEventStream> {
6950            self.requests.lock().unwrap().push(request);
6951            let mut calls = self.calls.lock().unwrap();
6952            *calls += 1;
6953            let events = match *calls {
6954                1 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
6955                    id: "ledger-open".to_string(),
6956                    name: TASK_LEDGER_TOOL_NAME.to_string(),
6957                    arguments: serde_json::json!({
6958                        "tasks": [
6959                            {
6960                                "id": "inspect",
6961                                "content": "Inspect local assets",
6962                                "status": "completed",
6963                                "evidence": "listed workspace"
6964                            },
6965                            {
6966                                "id": "write",
6967                                "content": "Write /app/result.txt",
6968                                "status": "pending"
6969                            }
6970                        ],
6971                        "requireCompletionEvidence": true
6972                    })
6973                    .to_string(),
6974                }))],
6975                3 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
6976                    id: "ledger-complete".to_string(),
6977                    name: TASK_LEDGER_TOOL_NAME.to_string(),
6978                    arguments: serde_json::json!({
6979                        "tasks": [
6980                            {
6981                                "id": "inspect",
6982                                "content": "Inspect local assets",
6983                                "status": "completed",
6984                                "evidence": "listed workspace"
6985                            },
6986                            {
6987                                "id": "write",
6988                                "content": "Write /app/result.txt",
6989                                "status": "completed",
6990                                "evidence": "wrote answer"
6991                            }
6992                        ],
6993                        "requireCompletionEvidence": true
6994                    })
6995                    .to_string(),
6996                }))],
6997                _ => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
6998                    text: "final".to_string(),
6999                    phase: None,
7000                }))],
7001            };
7002            Ok(Box::pin(stream::iter(events.into_iter().chain(
7003                std::iter::once(Ok(InferenceEvent::Completed(CompletionMetadata {
7004                    stop_reason: Some("stop".to_string()),
7005                    provider_response_id: None,
7006                }))),
7007            ))))
7008        }
7009    }
7010
7011    struct VerificationGateEngine {
7012        calls: StdMutex<u32>,
7013    }
7014
7015    #[async_trait::async_trait]
7016    impl InferenceEngine for VerificationGateEngine {
7017        fn id(&self) -> String {
7018            roder_api::catalog::PROVIDER_MOCK.to_string()
7019        }
7020
7021        fn capabilities(&self) -> InferenceCapabilities {
7022            InferenceCapabilities::coding_agent_default()
7023        }
7024
7025        async fn list_models(
7026            &self,
7027            _ctx: InferenceProviderContext<'_>,
7028        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7029            Ok(roder_api::catalog::models_for_provider(
7030                roder_api::catalog::PROVIDER_MOCK,
7031                true,
7032            ))
7033        }
7034
7035        async fn stream_turn(
7036            &self,
7037            _ctx: InferenceTurnContext<'_>,
7038            request: AgentInferenceRequest,
7039        ) -> anyhow::Result<InferenceEventStream> {
7040            let mut calls = self.calls.lock().unwrap();
7041            *calls += 1;
7042            let events = match *calls {
7043                1 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7044                    id: "write-1".to_string(),
7045                    name: "write_file".to_string(),
7046                    arguments: serde_json::json!({
7047                        "path": "src/lib.rs",
7048                        "content": "pub fn answer() -> u8 { 42 }\n"
7049                    })
7050                    .to_string(),
7051                }))],
7052                2 => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
7053                    text: "done too early".to_string(),
7054                    phase: None,
7055                }))],
7056                3 if request.transcript.iter().any(|item| {
7057                    matches!(
7058                        item,
7059                        TranscriptItem::UserMessage(message)
7060                            if message.text.contains("Verification gate blocked final completion")
7061                    )
7062                }) =>
7063                {
7064                    vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7065                        id: "verify-1".to_string(),
7066                        name: crate::verification_gate::VERIFICATION_TOOL_NAME.to_string(),
7067                        arguments: serde_json::json!({
7068                            "originalTask": "write code",
7069                            "changedFiles": ["src/lib.rs"],
7070                            "toolEvidence": ["write_file wrote src/lib.rs"],
7071                            "testsRun": ["cargo test -p roder-core verification_gate"],
7072                            "openGaps": [],
7073                            "status": "completed"
7074                        })
7075                        .to_string(),
7076                    }))]
7077                }
7078                _ => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
7079                    text: "verified final".to_string(),
7080                    phase: None,
7081                }))],
7082            };
7083            Ok(Box::pin(stream::iter(events.into_iter().chain(
7084                std::iter::once(Ok(InferenceEvent::Completed(CompletionMetadata {
7085                    stop_reason: Some("stop".to_string()),
7086                    provider_response_id: None,
7087                }))),
7088            ))))
7089        }
7090    }
7091
7092    struct SpeedPolicyEngine {
7093        calls: StdMutex<u32>,
7094        requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
7095    }
7096
7097    #[async_trait::async_trait]
7098    impl InferenceEngine for SpeedPolicyEngine {
7099        fn id(&self) -> String {
7100            roder_api::catalog::PROVIDER_MOCK.to_string()
7101        }
7102
7103        fn capabilities(&self) -> InferenceCapabilities {
7104            InferenceCapabilities::coding_agent_default()
7105        }
7106
7107        async fn list_models(
7108            &self,
7109            _ctx: InferenceProviderContext<'_>,
7110        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7111            Ok(roder_api::catalog::models_for_provider(
7112                roder_api::catalog::PROVIDER_MOCK,
7113                true,
7114            ))
7115        }
7116
7117        async fn stream_turn(
7118            &self,
7119            _ctx: InferenceTurnContext<'_>,
7120            request: AgentInferenceRequest,
7121        ) -> anyhow::Result<InferenceEventStream> {
7122            self.requests.lock().unwrap().push(request.clone());
7123            let mut calls = self.calls.lock().unwrap();
7124            *calls += 1;
7125            let events = match *calls {
7126                1 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7127                    id: "write-1".to_string(),
7128                    name: "write_file".to_string(),
7129                    arguments: serde_json::json!({
7130                        "path": "src/lib.rs",
7131                        "content": "pub fn answer() -> u8 { 42 }\n"
7132                    })
7133                    .to_string(),
7134                }))],
7135                3 if request.transcript.iter().any(|item| {
7136                    matches!(
7137                        item,
7138                        TranscriptItem::UserMessage(message)
7139                            if message.text.contains("Verification gate blocked final completion")
7140                    )
7141                }) =>
7142                {
7143                    vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7144                        id: "verify-1".to_string(),
7145                        name: crate::verification_gate::VERIFICATION_TOOL_NAME.to_string(),
7146                        arguments: serde_json::json!({
7147                            "originalTask": "write code",
7148                            "changedFiles": ["src/lib.rs"],
7149                            "toolEvidence": ["write_file wrote src/lib.rs"],
7150                            "testsRun": ["cargo test -p roder-core speed_policy"],
7151                            "openGaps": [],
7152                            "status": "completed"
7153                        })
7154                        .to_string(),
7155                    }))]
7156                }
7157                _ => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
7158                    text: "done".to_string(),
7159                    phase: None,
7160                }))],
7161            };
7162            Ok(Box::pin(stream::iter(events.into_iter().chain(
7163                std::iter::once(Ok(InferenceEvent::Completed(CompletionMetadata {
7164                    stop_reason: Some("stop".to_string()),
7165                    provider_response_id: None,
7166                }))),
7167            ))))
7168        }
7169    }
7170
7171    struct SwitchCaptureEngine {
7172        requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
7173    }
7174
7175    #[async_trait::async_trait]
7176    impl InferenceEngine for SwitchCaptureEngine {
7177        fn id(&self) -> String {
7178            roder_api::catalog::PROVIDER_MOCK.to_string()
7179        }
7180
7181        fn capabilities(&self) -> InferenceCapabilities {
7182            InferenceCapabilities::coding_agent_default()
7183        }
7184
7185        async fn list_models(
7186            &self,
7187            _ctx: InferenceProviderContext<'_>,
7188        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7189            Ok(roder_api::catalog::models_for_provider(
7190                roder_api::catalog::PROVIDER_MOCK,
7191                true,
7192            ))
7193        }
7194
7195        async fn stream_turn(
7196            &self,
7197            _ctx: InferenceTurnContext<'_>,
7198            request: AgentInferenceRequest,
7199        ) -> anyhow::Result<InferenceEventStream> {
7200            self.requests.lock().unwrap().push(request);
7201            Ok(Box::pin(stream::iter(vec![
7202                Ok(InferenceEvent::MessageDelta(MessageDelta {
7203                    text: "done".to_string(),
7204                    phase: None,
7205                })),
7206                Ok(InferenceEvent::Completed(CompletionMetadata {
7207                    stop_reason: Some("stop".to_string()),
7208                    provider_response_id: None,
7209                })),
7210            ])))
7211        }
7212    }
7213
7214    struct DeadlineEngine;
7215
7216    #[async_trait::async_trait]
7217    impl InferenceEngine for DeadlineEngine {
7218        fn id(&self) -> String {
7219            roder_api::catalog::PROVIDER_MOCK.to_string()
7220        }
7221
7222        fn capabilities(&self) -> InferenceCapabilities {
7223            InferenceCapabilities::coding_agent_default()
7224        }
7225
7226        async fn list_models(
7227            &self,
7228            _ctx: InferenceProviderContext<'_>,
7229        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7230            Ok(Vec::new())
7231        }
7232
7233        async fn stream_turn(
7234            &self,
7235            _ctx: InferenceTurnContext<'_>,
7236            _request: AgentInferenceRequest,
7237        ) -> anyhow::Result<InferenceEventStream> {
7238            Ok(Box::pin(stream::once(async {
7239                tokio::time::sleep(std::time::Duration::from_secs(60)).await;
7240                Ok(InferenceEvent::MessageDelta(MessageDelta {
7241                    text: "too late".to_string(),
7242                    phase: None,
7243                }))
7244            })))
7245        }
7246    }
7247
7248    struct WriteFileContributor;
7249
7250    impl ToolContributor for WriteFileContributor {
7251        fn id(&self) -> String {
7252            "test-write".to_string()
7253        }
7254
7255        fn contribute(&self, registry: &mut ToolRegistry) -> anyhow::Result<()> {
7256            registry.register(Arc::new(WriteFileTool))
7257        }
7258    }
7259
7260    struct WriteFileTool;
7261
7262    #[async_trait::async_trait]
7263    impl ToolExecutor for WriteFileTool {
7264        fn spec(&self) -> ToolSpec {
7265            ToolSpec {
7266                name: "write_file".to_string(),
7267                description: "Write a test file.".to_string(),
7268                parameters: serde_json::json!({
7269                    "type": "object",
7270                    "properties": {
7271                        "path": { "type": "string" },
7272                        "content": { "type": "string" }
7273                    },
7274                    "required": ["path", "content"],
7275                    "additionalProperties": false
7276                }),
7277            }
7278        }
7279
7280        async fn execute(
7281            &self,
7282            _ctx: ToolExecutionContext,
7283            call: ToolCall,
7284        ) -> anyhow::Result<ToolResult> {
7285            let path = call
7286                .arguments
7287                .get("path")
7288                .and_then(serde_json::Value::as_str)
7289                .unwrap_or("src/lib.rs");
7290            Ok(ToolResult {
7291                id: call.id,
7292                name: call.name,
7293                text: format!("wrote {path}"),
7294                data: serde_json::json!({ "path": path }),
7295                is_error: false,
7296            })
7297        }
7298    }
7299
7300    struct ProfileToolContributor;
7301
7302    impl ToolContributor for ProfileToolContributor {
7303        fn id(&self) -> String {
7304            "profile-tools".to_string()
7305        }
7306
7307        fn contribute(&self, registry: &mut ToolRegistry) -> anyhow::Result<()> {
7308            for name in ["apply_patch", "edit", "multi_edit", "write_file"] {
7309                registry.register(Arc::new(ProfileTool {
7310                    name: name.to_string(),
7311                }))?;
7312            }
7313            Ok(())
7314        }
7315    }
7316
7317    struct ProfileTool {
7318        name: String,
7319    }
7320
7321    #[async_trait::async_trait]
7322    impl ToolExecutor for ProfileTool {
7323        fn spec(&self) -> ToolSpec {
7324            ToolSpec {
7325                name: self.name.clone(),
7326                description: format!("{} test tool", self.name),
7327                parameters: serde_json::json!({
7328                    "type": "object",
7329                    "properties": {
7330                        "path": { "type": "string" },
7331                        "content": { "type": "string" }
7332                    },
7333                    "required": ["path", "content"],
7334                    "additionalProperties": false
7335                }),
7336            }
7337        }
7338
7339        async fn execute(
7340            &self,
7341            _ctx: ToolExecutionContext,
7342            call: ToolCall,
7343        ) -> anyhow::Result<ToolResult> {
7344            Ok(ToolResult {
7345                id: call.id,
7346                name: call.name,
7347                text: "ok".to_string(),
7348                data: serde_json::json!({}),
7349                is_error: false,
7350            })
7351        }
7352    }
7353
7354    fn test_model_profile(model: &str) -> ModelHarnessProfile {
7355        ModelHarnessProfile {
7356            model: model.to_string(),
7357            provider: roder_api::catalog::PROVIDER_OPENAI.to_string(),
7358            provider_family: ProviderFamily::OpenAi,
7359            edit_tool: Some(EDIT_TOOL_EDIT.to_string()),
7360            schema_policy: ModelSchemaPolicy::StandardRequiredFirst,
7361            instruction_overlay: ModelInstructionOverlay::IntuitiveContext,
7362            reasoning: ModelProfileReasoning {
7363                orientation: Some(REASONING_LOW.to_string()),
7364                execution: Some(REASONING_LOW.to_string()),
7365                verification: Some(REASONING_LOW.to_string()),
7366                recovery: Some(REASONING_LOW.to_string()),
7367            },
7368            parallel_tool_calls: Some(false),
7369            auto_compact_token_limit: Some(123_000),
7370        }
7371    }
7372
7373    async fn captured_profile_request(cfg: RuntimeConfig) -> AgentInferenceRequest {
7374        let captured = Arc::new(StdMutex::new(None));
7375        let mut builder = ExtensionRegistryBuilder::new();
7376        builder.inference_engine(Arc::new(CapturingEngine {
7377            request: captured.clone(),
7378        }));
7379        builder.tool_contributor(Arc::new(ProfileToolContributor));
7380        let runtime = Arc::new(Runtime::new(builder.build().unwrap(), cfg).unwrap());
7381        let mut rx = runtime.subscribe_events();
7382        let turn_id = runtime
7383            .start_turn(StartTurnRequest {
7384                thread_id: "thread-model-profile".to_string(),
7385                message: "use profile knobs".to_string(),
7386                images: Vec::new(),
7387                provider_override: None,
7388                model_override: None,
7389                reasoning_override: None,
7390                workspace: test_workspace(),
7391                instructions: InstructionBundle {
7392                    system: None,
7393                    developer: Some("base developer".to_string()),
7394                    developer_context: None,
7395                },
7396                developer_context: None,
7397                task_ledger_required: false,
7398                service_tier_override: None,
7399            })
7400            .await
7401            .unwrap();
7402
7403        tokio::time::timeout(std::time::Duration::from_secs(5), async {
7404            loop {
7405                let envelope = rx.recv().await.unwrap();
7406                if envelope.turn_id.as_deref() != Some(&turn_id) {
7407                    continue;
7408                }
7409                match envelope.event {
7410                    RoderEvent::TurnCompleted(_) => break,
7411                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7412                    _ => {}
7413                }
7414            }
7415        })
7416        .await
7417        .unwrap();
7418
7419        captured.lock().unwrap().clone().unwrap()
7420    }
7421
7422    struct ToolThenStopEngine {
7423        calls: StdMutex<u32>,
7424    }
7425
7426    #[async_trait::async_trait]
7427    impl InferenceEngine for ToolThenStopEngine {
7428        fn id(&self) -> String {
7429            roder_api::catalog::PROVIDER_MOCK.to_string()
7430        }
7431
7432        fn capabilities(&self) -> InferenceCapabilities {
7433            InferenceCapabilities::coding_agent_default()
7434        }
7435
7436        async fn list_models(
7437            &self,
7438            _ctx: InferenceProviderContext<'_>,
7439        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7440            Ok(roder_api::catalog::models_for_provider(
7441                roder_api::catalog::PROVIDER_MOCK,
7442                true,
7443            ))
7444        }
7445
7446        async fn stream_turn(
7447            &self,
7448            _ctx: InferenceTurnContext<'_>,
7449            _request: AgentInferenceRequest,
7450        ) -> anyhow::Result<InferenceEventStream> {
7451            let mut calls = self.calls.lock().unwrap();
7452            *calls += 1;
7453            let events = match *calls {
7454                1 => vec![
7455                    Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7456                        id: "write-1".to_string(),
7457                        name: "write_file".to_string(),
7458                        arguments: serde_json::json!({
7459                            "path": "src/lib.rs",
7460                            "content": "pub fn answer() -> u8 { 42 }\n"
7461                        })
7462                        .to_string(),
7463                    })),
7464                    Ok(InferenceEvent::Completed(CompletionMetadata {
7465                        stop_reason: Some("tool_use".to_string()),
7466                        provider_response_id: None,
7467                    })),
7468                ],
7469                _ => vec![
7470                    Ok(InferenceEvent::MessageDelta(MessageDelta {
7471                        text: "final".to_string(),
7472                        phase: None,
7473                    })),
7474                    Ok(InferenceEvent::Completed(CompletionMetadata {
7475                        stop_reason: Some("end_turn".to_string()),
7476                        provider_response_id: None,
7477                    })),
7478                ],
7479            };
7480            Ok(Box::pin(stream::iter(events)))
7481        }
7482    }
7483
7484    #[tokio::test]
7485    async fn turn_completed_reports_terminal_step_finish_reason() {
7486        let mut builder = ExtensionRegistryBuilder::new();
7487        builder.inference_engine(Arc::new(ToolThenStopEngine {
7488            calls: StdMutex::new(0),
7489        }));
7490        builder.tool_contributor(Arc::new(WriteFileContributor));
7491        let runtime = Arc::new(
7492            Runtime::new(
7493                builder.build().unwrap(),
7494                RuntimeConfig {
7495                    policy_mode: PolicyMode::Bypass,
7496                    agent_swarm_mode: false,
7497                    ultra_mode: false,
7498                    ..RuntimeConfig::default()
7499                },
7500            )
7501            .unwrap(),
7502        );
7503        let mut rx = runtime.subscribe_events();
7504        let turn_id = runtime
7505            .start_turn(StartTurnRequest {
7506                thread_id: "thread-finish-reason".to_string(),
7507                message: "write then finish".to_string(),
7508                images: Vec::new(),
7509                provider_override: None,
7510                model_override: None,
7511                reasoning_override: None,
7512                workspace: test_workspace(),
7513                instructions: InstructionBundle {
7514                    system: None,
7515                    developer: None,
7516                    developer_context: None,
7517                },
7518                developer_context: None,
7519                task_ledger_required: false,
7520                service_tier_override: None,
7521            })
7522            .await
7523            .unwrap();
7524
7525        let completed = tokio::time::timeout(std::time::Duration::from_secs(5), async {
7526            loop {
7527                let envelope = rx.recv().await.unwrap();
7528                if envelope.turn_id.as_deref() != Some(&turn_id) {
7529                    continue;
7530                }
7531                match envelope.event {
7532                    RoderEvent::TurnCompleted(event) => break event,
7533                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7534                    _ => {}
7535                }
7536            }
7537        })
7538        .await
7539        .unwrap();
7540
7541        // The mid-turn tool_use stop reason must not leak; the terminal
7542        // end_turn step decides the turn's finish reason.
7543        assert_eq!(completed.finish_reason.as_deref(), Some("stop"));
7544    }
7545
7546    #[tokio::test]
7547    async fn inference_router_selection_changes_request_model_and_records_event() {
7548        let default_requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7549        let routed_requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7550        let contexts = Arc::new(StdMutex::new(Vec::<InferenceRoutingContext>::new()));
7551        let selected = ModelSelection {
7552            provider: "routed-provider".to_string(),
7553            model: "routed-model".to_string(),
7554        };
7555        let default = ModelSelection {
7556            provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7557            model: "mock".to_string(),
7558        };
7559        let decision = InferenceRoutingDecision {
7560            reasoning: Some(ReasoningConfig {
7561                enabled: true,
7562                level: Some(REASONING_LOW.to_string()),
7563            }),
7564            confidence: Some(0.91),
7565            baseline: Some(default.clone()),
7566            matched_signals: vec![roder_api::inference_routing::InferenceRoutingSignal::new(
7567                "intent", "routine",
7568            )],
7569            ..InferenceRoutingDecision::selected("test-router", selected.clone(), "routine request")
7570        };
7571
7572        let mut builder = ExtensionRegistryBuilder::new();
7573        builder.inference_engine(Arc::new(RoutingCaptureEngine {
7574            id: roder_api::catalog::PROVIDER_MOCK,
7575            models: vec![routing_test_model("mock", &[REASONING_LOW])],
7576            requests: default_requests.clone(),
7577        }));
7578        builder.inference_engine(Arc::new(RoutingCaptureEngine {
7579            id: "routed-provider",
7580            models: vec![routing_test_model(
7581                "routed-model",
7582                &[REASONING_LOW, REASONING_MEDIUM],
7583            )],
7584            requests: routed_requests.clone(),
7585        }));
7586        builder.inference_router(Arc::new(StaticRouter {
7587            id: "test-router",
7588            decision,
7589            contexts: contexts.clone(),
7590        }));
7591        let thread_root =
7592            std::env::temp_dir().join(format!("roder-routing-auto-{}", uuid::Uuid::new_v4()));
7593        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
7594            base_path: thread_root.clone(),
7595        }));
7596        let runtime = Arc::new(
7597            Runtime::new(
7598                builder.build().unwrap(),
7599                RuntimeConfig {
7600                    default_provider: default.provider.clone(),
7601                    default_model: default.model.clone(),
7602                    ..RuntimeConfig::default()
7603                },
7604            )
7605            .unwrap(),
7606        );
7607        let thread_id = runtime
7608            .create_thread_with(CreateThreadRequest {
7609                title: Some("Routing auto".to_string()),
7610                workspace: test_workspace(),
7611                workspace_id: None,
7612                root_id: None,
7613                provider: Some(default.provider.clone()),
7614                model: Some(default.model.clone()),
7615                tool_allowlist: Vec::new(),
7616                developer_instructions: None,
7617                external_tools: Vec::new(),
7618                selection_mode: Some(ModelSelectionMode::auto(
7619                    "test-router:coding",
7620                    "test-router",
7621                    "Auto: Coding",
7622                    default.clone(),
7623                    Some("coding".to_string()),
7624                    None,
7625                )),
7626                runner: None,
7627            })
7628            .await
7629            .unwrap()
7630            .thread_id;
7631        let mut rx = runtime.subscribe_events();
7632        let turn_id = runtime
7633            .start_turn(StartTurnRequest {
7634                thread_id: thread_id.clone(),
7635                message: "small cleanup".to_string(),
7636                images: Vec::new(),
7637                provider_override: None,
7638                model_override: None,
7639                reasoning_override: None,
7640                workspace: test_workspace(),
7641                instructions: InstructionBundle::default(),
7642                developer_context: None,
7643                task_ledger_required: false,
7644                service_tier_override: None,
7645            })
7646            .await
7647            .unwrap();
7648
7649        let mut routing_event = None;
7650        let mut inference_started = None;
7651        tokio::time::timeout(std::time::Duration::from_secs(5), async {
7652            loop {
7653                let envelope = rx.recv().await.unwrap();
7654                if envelope.turn_id.as_deref() != Some(&turn_id) {
7655                    continue;
7656                }
7657                match envelope.event {
7658                    RoderEvent::InferenceRoutingDecision(event) => {
7659                        routing_event = Some(event);
7660                    }
7661                    RoderEvent::InferenceStarted(event) => {
7662                        inference_started = Some(event);
7663                    }
7664                    RoderEvent::TurnCompleted(_) => break,
7665                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7666                    _ => {}
7667                }
7668            }
7669        })
7670        .await
7671        .unwrap();
7672
7673        assert!(default_requests.lock().unwrap().is_empty());
7674        let routed_requests = routed_requests.lock().unwrap();
7675        assert_eq!(routed_requests.len(), 1);
7676        assert_eq!(routed_requests[0].model, selected);
7677        assert_eq!(
7678            routed_requests[0].reasoning.level.as_deref(),
7679            Some(REASONING_LOW)
7680        );
7681        assert_eq!(
7682            routed_requests[0].metadata["inferenceRouting"]["outcome"],
7683            "selected"
7684        );
7685
7686        let routing_event = routing_event.expect("routing decision event");
7687        assert_eq!(routing_event.default_selection, default);
7688        assert_eq!(routing_event.selected_selection, selected);
7689        assert_eq!(
7690            routing_event.decision.outcome,
7691            InferenceRoutingOutcome::Selected
7692        );
7693        assert_eq!(
7694            inference_started.expect("inference started event").model,
7695            selected
7696        );
7697
7698        let contexts = contexts.lock().unwrap();
7699        assert_eq!(contexts.len(), 1);
7700        assert_eq!(contexts[0].default_selection, default);
7701        assert_eq!(contexts[0].candidates.len(), 2);
7702        assert!(
7703            contexts[0]
7704                .signals
7705                .iter()
7706                .any(|signal| signal.key == "profile" && signal.value == "coding")
7707        );
7708        let _ = std::fs::remove_dir_all(thread_root);
7709    }
7710
7711    #[tokio::test]
7712    async fn inference_router_is_bypassed_for_explicit_selection() {
7713        let requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7714        let contexts = Arc::new(StdMutex::new(Vec::<InferenceRoutingContext>::new()));
7715        let selected = ModelSelection {
7716            provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7717            model: "mock".to_string(),
7718        };
7719
7720        let mut builder = ExtensionRegistryBuilder::new();
7721        builder.inference_engine(Arc::new(RoutingCaptureEngine {
7722            id: roder_api::catalog::PROVIDER_MOCK,
7723            models: vec![routing_test_model("mock", &[REASONING_LOW])],
7724            requests: requests.clone(),
7725        }));
7726        builder.inference_router(Arc::new(StaticRouter {
7727            id: "test-router",
7728            decision: InferenceRoutingDecision::selected(
7729                "test-router",
7730                ModelSelection {
7731                    provider: "missing".to_string(),
7732                    model: "missing".to_string(),
7733                },
7734                "would route if called",
7735            ),
7736            contexts: contexts.clone(),
7737        }));
7738        let thread_root =
7739            std::env::temp_dir().join(format!("roder-routing-explicit-{}", uuid::Uuid::new_v4()));
7740        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
7741            base_path: thread_root.clone(),
7742        }));
7743        let runtime = Arc::new(
7744            Runtime::new(
7745                builder.build().unwrap(),
7746                RuntimeConfig {
7747                    default_provider: selected.provider.clone(),
7748                    default_model: selected.model.clone(),
7749                    inference_router: RuntimeInferenceRouterConfig {
7750                        enabled: true,
7751                        router_id: Some("test-router".to_string()),
7752                    },
7753                    ..RuntimeConfig::default()
7754                },
7755            )
7756            .unwrap(),
7757        );
7758        let thread_id = runtime
7759            .create_thread_with(CreateThreadRequest {
7760                title: Some("Routing explicit".to_string()),
7761                workspace: test_workspace(),
7762                workspace_id: None,
7763                root_id: None,
7764                provider: Some(selected.provider.clone()),
7765                model: Some(selected.model.clone()),
7766                tool_allowlist: Vec::new(),
7767                developer_instructions: None,
7768                external_tools: Vec::new(),
7769                selection_mode: Some(ModelSelectionMode::auto(
7770                    "test-router:default",
7771                    "test-router",
7772                    "Auto",
7773                    selected.clone(),
7774                    None,
7775                    None,
7776                )),
7777                runner: None,
7778            })
7779            .await
7780            .unwrap()
7781            .thread_id;
7782        let mut rx = runtime.subscribe_events();
7783        let turn_id = runtime
7784            .start_turn(StartTurnRequest {
7785                thread_id,
7786                message: "use explicit selection".to_string(),
7787                images: Vec::new(),
7788                provider_override: Some(selected.provider.clone()),
7789                model_override: Some(selected.model.clone()),
7790                reasoning_override: None,
7791                workspace: test_workspace(),
7792                instructions: InstructionBundle::default(),
7793                developer_context: None,
7794                task_ledger_required: false,
7795                service_tier_override: None,
7796            })
7797            .await
7798            .unwrap();
7799
7800        let mut saw_routing_event = false;
7801        tokio::time::timeout(std::time::Duration::from_secs(5), async {
7802            loop {
7803                let envelope = rx.recv().await.unwrap();
7804                if envelope.turn_id.as_deref() != Some(&turn_id) {
7805                    continue;
7806                }
7807                match envelope.event {
7808                    RoderEvent::InferenceRoutingDecision(_) => {
7809                        saw_routing_event = true;
7810                    }
7811                    RoderEvent::TurnCompleted(_) => break,
7812                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7813                    _ => {}
7814                }
7815            }
7816        })
7817        .await
7818        .unwrap();
7819
7820        assert!(!saw_routing_event);
7821        assert!(contexts.lock().unwrap().is_empty());
7822        let requests = requests.lock().unwrap();
7823        assert_eq!(requests.len(), 1);
7824        assert_eq!(requests[0].model, selected);
7825        let _ = std::fs::remove_dir_all(thread_root);
7826    }
7827
7828    #[tokio::test]
7829    async fn inference_router_is_bypassed_for_manual_selection_mode() {
7830        let requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7831        let contexts = Arc::new(StdMutex::new(Vec::<InferenceRoutingContext>::new()));
7832        let selected = ModelSelection {
7833            provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7834            model: "mock".to_string(),
7835        };
7836
7837        let mut builder = ExtensionRegistryBuilder::new();
7838        builder.inference_engine(Arc::new(RoutingCaptureEngine {
7839            id: roder_api::catalog::PROVIDER_MOCK,
7840            models: vec![routing_test_model("mock", &[REASONING_LOW])],
7841            requests: requests.clone(),
7842        }));
7843        builder.inference_router(Arc::new(StaticRouter {
7844            id: "test-router",
7845            decision: InferenceRoutingDecision::selected(
7846                "test-router",
7847                ModelSelection {
7848                    provider: "missing".to_string(),
7849                    model: "missing".to_string(),
7850                },
7851                "would route if called",
7852            ),
7853            contexts: contexts.clone(),
7854        }));
7855        let thread_root =
7856            std::env::temp_dir().join(format!("roder-routing-manual-{}", uuid::Uuid::new_v4()));
7857        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
7858            base_path: thread_root.clone(),
7859        }));
7860        let runtime = Arc::new(
7861            Runtime::new(
7862                builder.build().unwrap(),
7863                RuntimeConfig {
7864                    default_provider: selected.provider.clone(),
7865                    default_model: selected.model.clone(),
7866                    inference_router: RuntimeInferenceRouterConfig {
7867                        enabled: true,
7868                        router_id: Some("test-router".to_string()),
7869                    },
7870                    ..RuntimeConfig::default()
7871                },
7872            )
7873            .unwrap(),
7874        );
7875        let thread_id = runtime
7876            .create_thread_with(CreateThreadRequest {
7877                title: Some("Routing manual".to_string()),
7878                workspace: test_workspace(),
7879                workspace_id: None,
7880                root_id: None,
7881                provider: Some(selected.provider.clone()),
7882                model: Some(selected.model.clone()),
7883                tool_allowlist: Vec::new(),
7884                developer_instructions: None,
7885                external_tools: Vec::new(),
7886                selection_mode: Some(ModelSelectionMode::manual(
7887                    selected.provider.clone(),
7888                    selected.model.clone(),
7889                    None,
7890                )),
7891                runner: None,
7892            })
7893            .await
7894            .unwrap()
7895            .thread_id;
7896        let mut rx = runtime.subscribe_events();
7897        let turn_id = runtime
7898            .start_turn(StartTurnRequest {
7899                thread_id,
7900                message: "use selected manual model".to_string(),
7901                images: Vec::new(),
7902                provider_override: None,
7903                model_override: None,
7904                reasoning_override: None,
7905                workspace: test_workspace(),
7906                instructions: InstructionBundle::default(),
7907                developer_context: None,
7908                task_ledger_required: false,
7909                service_tier_override: None,
7910            })
7911            .await
7912            .unwrap();
7913
7914        let mut saw_routing_event = false;
7915        tokio::time::timeout(std::time::Duration::from_secs(5), async {
7916            loop {
7917                let envelope = rx.recv().await.unwrap();
7918                if envelope.turn_id.as_deref() != Some(&turn_id) {
7919                    continue;
7920                }
7921                match envelope.event {
7922                    RoderEvent::InferenceRoutingDecision(_) => {
7923                        saw_routing_event = true;
7924                    }
7925                    RoderEvent::TurnCompleted(_) => break,
7926                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7927                    _ => {}
7928                }
7929            }
7930        })
7931        .await
7932        .unwrap();
7933
7934        assert!(!saw_routing_event);
7935        assert!(contexts.lock().unwrap().is_empty());
7936        let requests = requests.lock().unwrap();
7937        assert_eq!(requests.len(), 1);
7938        assert_eq!(requests[0].model, selected);
7939        let _ = std::fs::remove_dir_all(thread_root);
7940    }
7941
7942    #[test]
7943    fn enabled_inference_router_requires_registered_router() {
7944        let mut builder = ExtensionRegistryBuilder::new();
7945        builder.inference_engine(Arc::new(FakeInferenceEngine));
7946
7947        let err = match Runtime::new(
7948            builder.build().unwrap(),
7949            RuntimeConfig {
7950                inference_router: RuntimeInferenceRouterConfig {
7951                    enabled: true,
7952                    router_id: Some("missing-router".to_string()),
7953                },
7954                ..RuntimeConfig::default()
7955            },
7956        ) {
7957            Ok(_) => panic!("runtime should reject unknown inference router"),
7958            Err(err) => err,
7959        };
7960
7961        assert!(
7962            err.to_string()
7963                .contains("inference router \"missing-router\" is not registered")
7964        );
7965    }
7966
7967    #[tokio::test]
7968    async fn model_profile_routes_request_knobs_to_next_inference() {
7969        let request = captured_profile_request(RuntimeConfig {
7970            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7971            default_model: "gpt-5.5".to_string(),
7972            model_profiles: std::collections::HashMap::from([(
7973                "gpt-5.5".to_string(),
7974                test_model_profile("gpt-5.5"),
7975            )]),
7976            ..RuntimeConfig::default()
7977        })
7978        .await;
7979
7980        let tool_names = request
7981            .tools
7982            .iter()
7983            .map(|tool| tool.name.as_str())
7984            .collect::<Vec<_>>();
7985        assert!(tool_names.contains(&"apply_patch"));
7986        assert!(tool_names.contains(&"edit"));
7987        assert!(tool_names.contains(&"multi_edit"));
7988        assert!(tool_names.contains(&"write_file"));
7989        assert_eq!(request.reasoning.level.as_deref(), Some(REASONING_LOW));
7990        assert_eq!(request.runtime.parallel_tool_calls, Some(false));
7991        assert_eq!(request.runtime.auto_compact_token_limit, Some(123_000));
7992        assert!(
7993            request
7994                .instructions
7995                .developer
7996                .as_deref()
7997                .unwrap_or_default()
7998                .contains("Use the provided context as the current working set")
7999        );
8000        assert_eq!(
8001            request
8002                .metadata
8003                .pointer("/modelProfile/schemaPolicy")
8004                .and_then(serde_json::Value::as_str),
8005            Some("standard_required_first")
8006        );
8007    }
8008
8009    #[tokio::test]
8010    async fn ultra_reasoning_reaches_transport_state_and_enables_proactive_delegation() {
8011        let request = captured_profile_request(RuntimeConfig {
8012            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8013            default_model: "gpt-5.6-sol".to_string(),
8014            reasoning: Some(REASONING_ULTRA.to_string()),
8015            ..RuntimeConfig::default()
8016        })
8017        .await;
8018
8019        assert_eq!(request.reasoning.level.as_deref(), Some(REASONING_ULTRA));
8020        let developer = request
8021            .instructions
8022            .developer
8023            .as_deref()
8024            .expect("ultra developer instructions");
8025        assert!(developer.contains("Proactive multi-agent delegation is active"));
8026    }
8027
8028    #[tokio::test]
8029    async fn lower_sol_effort_requires_explicit_multi_agent_request() {
8030        let request = captured_profile_request(RuntimeConfig {
8031            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8032            default_model: "gpt-5.6-sol".to_string(),
8033            reasoning: Some(roder_api::catalog::REASONING_MEDIUM.to_string()),
8034            ..RuntimeConfig::default()
8035        })
8036        .await;
8037
8038        let developer = request
8039            .instructions
8040            .developer
8041            .as_deref()
8042            .expect("sol developer instructions");
8043        assert!(developer.contains("Do not spawn sub-agents unless"));
8044        assert!(!developer.contains("Proactive multi-agent delegation is active"));
8045    }
8046
8047    #[tokio::test]
8048    async fn luna_does_not_receive_codex_v2_multi_agent_policy() {
8049        let request = captured_profile_request(RuntimeConfig {
8050            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8051            default_model: "gpt-5.6-luna".to_string(),
8052            reasoning: Some(roder_api::catalog::REASONING_MAX.to_string()),
8053            ..RuntimeConfig::default()
8054        })
8055        .await;
8056
8057        let developer = request
8058            .instructions
8059            .developer
8060            .as_deref()
8061            .unwrap_or_default();
8062        assert!(!developer.contains("Do not spawn sub-agents unless"));
8063        assert!(!developer.contains("Proactive multi-agent delegation is active"));
8064    }
8065
8066    #[tokio::test]
8067    async fn ultra_mode_enables_proactive_multi_agent_for_any_model() {
8068        // Luna has no Ultra effort, but ultra mode still injects proactive policy.
8069        let request = captured_profile_request(RuntimeConfig {
8070            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8071            default_model: "gpt-5.6-luna".to_string(),
8072            reasoning: Some(roder_api::catalog::REASONING_MAX.to_string()),
8073            ultra_mode: true,
8074            ..RuntimeConfig::default()
8075        })
8076        .await;
8077
8078        let developer = request
8079            .instructions
8080            .developer
8081            .as_deref()
8082            .expect("ultra mode developer instructions");
8083        assert!(developer.contains("Proactive multi-agent delegation is active"));
8084        assert!(developer.contains("spawn_agent"));
8085    }
8086
8087    #[tokio::test]
8088    async fn ultra_mode_overrides_explicit_request_only_on_sol() {
8089        // Medium Sol effort is normally explicit-request-only; ultra mode flips
8090        // it to proactive without requiring Ultra reasoning effort.
8091        let request = captured_profile_request(RuntimeConfig {
8092            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8093            default_model: "gpt-5.6-sol".to_string(),
8094            reasoning: Some(roder_api::catalog::REASONING_MEDIUM.to_string()),
8095            ultra_mode: true,
8096            ..RuntimeConfig::default()
8097        })
8098        .await;
8099
8100        let developer = request
8101            .instructions
8102            .developer
8103            .as_deref()
8104            .expect("ultra mode sol developer instructions");
8105        assert!(developer.contains("Proactive multi-agent delegation is active"));
8106        assert!(!developer.contains("Do not spawn sub-agents unless"));
8107    }
8108
8109    #[tokio::test]
8110    async fn turn_developer_context_reaches_inference_and_does_not_persist() {
8111        let captured = Arc::new(StdMutex::new(None));
8112        let mut builder = ExtensionRegistryBuilder::new();
8113        builder.inference_engine(Arc::new(CapturingEngine {
8114            request: captured.clone(),
8115        }));
8116        let runtime = Arc::new(
8117            Runtime::new(
8118                builder.build().unwrap(),
8119                RuntimeConfig {
8120                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8121                    default_model: "gpt-5.5".to_string(),
8122                    ..RuntimeConfig::default()
8123                },
8124            )
8125            .unwrap(),
8126        );
8127
8128        async fn run_turn(runtime: &Arc<Runtime>, developer_context: Option<String>) {
8129            let mut rx = runtime.subscribe_events();
8130            let turn_id = runtime
8131                .start_turn(StartTurnRequest {
8132                    thread_id: "thread-turn-context".to_string(),
8133                    message: "hello".to_string(),
8134                    images: Vec::new(),
8135                    provider_override: None,
8136                    model_override: None,
8137                    reasoning_override: None,
8138                    workspace: test_workspace(),
8139                    instructions: InstructionBundle::default(),
8140                    developer_context,
8141                    task_ledger_required: false,
8142                    service_tier_override: None,
8143                })
8144                .await
8145                .unwrap();
8146            tokio::time::timeout(std::time::Duration::from_secs(5), async {
8147                loop {
8148                    let envelope = rx.recv().await.unwrap();
8149                    if envelope.turn_id.as_deref() != Some(&turn_id) {
8150                        continue;
8151                    }
8152                    match envelope.event {
8153                        RoderEvent::TurnCompleted(_) => break,
8154                        RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8155                        _ => {}
8156                    }
8157                }
8158            })
8159            .await
8160            .unwrap();
8161        }
8162
8163        run_turn(
8164            &runtime,
8165            Some("Connected accounts: example-service.".to_string()),
8166        )
8167        .await;
8168        let request = captured.lock().unwrap().clone().unwrap();
8169        assert_eq!(
8170            request.instructions.developer_context.as_deref(),
8171            Some("Connected accounts: example-service.")
8172        );
8173
8174        // The context is per-turn only: the next turn on the same thread
8175        // without a developerContext must not see the previous one.
8176        run_turn(&runtime, None).await;
8177        let request = captured.lock().unwrap().clone().unwrap();
8178        assert_eq!(request.instructions.developer_context, None);
8179    }
8180
8181    #[tokio::test]
8182    async fn tool_search_overrides_route_to_next_inference_request() {
8183        let request = captured_profile_request(RuntimeConfig {
8184            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8185            default_model: "gpt-5.4".to_string(),
8186            tool_search: ToolSearchConfig {
8187                mode: roder_api::inference::ToolSearchMode::Auto,
8188                max_catalog_items: Some(100),
8189                ..ToolSearchConfig::default()
8190            },
8191            provider_tool_search: std::collections::HashMap::from([(
8192                roder_api::catalog::PROVIDER_MOCK.to_string(),
8193                roder_api::inference::ToolSearchConfigOverlay {
8194                    include_skills: Some(false),
8195                    provider_variant: Some(roder_api::inference::ToolSearchProviderVariant::Regex),
8196                    ..Default::default()
8197                },
8198            )]),
8199            model_tool_search: std::collections::HashMap::from([(
8200                "gpt-5.4".to_string(),
8201                roder_api::inference::ToolSearchConfigOverlay {
8202                    mode: Some(roder_api::inference::ToolSearchMode::ProviderNative),
8203                    max_catalog_items: Some(25),
8204                    provider_variant: Some(roder_api::inference::ToolSearchProviderVariant::Bm25),
8205                    ..Default::default()
8206                },
8207            )]),
8208            ..RuntimeConfig::default()
8209        })
8210        .await;
8211
8212        assert_eq!(
8213            request.runtime.tool_search.mode,
8214            roder_api::inference::ToolSearchMode::ProviderNative
8215        );
8216        assert_eq!(request.runtime.tool_search.max_catalog_items, Some(25));
8217        assert_eq!(
8218            request.runtime.tool_search.provider_variant,
8219            roder_api::inference::ToolSearchProviderVariant::Bm25
8220        );
8221    }
8222
8223    #[tokio::test]
8224    async fn context_entrypoint_hints_use_turn_workspace() {
8225        let process_workspace = runtime_test_workspace("entrypoint-process");
8226        let thread_workspace = runtime_test_workspace("entrypoint-thread");
8227        std::fs::create_dir_all(process_workspace.join("src")).unwrap();
8228        std::fs::create_dir_all(thread_workspace.join("src")).unwrap();
8229        std::fs::write(
8230            process_workspace.join("src/sidebar-thread-groups.ts"),
8231            "export const desktopLeak = true;\n",
8232        )
8233        .unwrap();
8234        std::fs::write(
8235            thread_workspace.join("src/voice-plan-feedback.ts"),
8236            "export const voicePlanFeedback = true;\n",
8237        )
8238        .unwrap();
8239
8240        let captured = Arc::new(StdMutex::new(None));
8241        let mut builder = ExtensionRegistryBuilder::new();
8242        builder.inference_engine(Arc::new(CapturingEngine {
8243            request: captured.clone(),
8244        }));
8245        builder.context_planner(Arc::new(roder_context::EntrypointContextPlanner::new(
8246            process_workspace.clone(),
8247        )));
8248        let runtime = Arc::new(
8249            Runtime::new(
8250                builder.build().unwrap(),
8251                RuntimeConfig {
8252                    workspace: Some(process_workspace.display().to_string()),
8253                    ..RuntimeConfig::default()
8254                },
8255            )
8256            .unwrap(),
8257        );
8258        let mut rx = runtime.subscribe_events();
8259
8260        let turn_id = runtime
8261            .start_turn(StartTurnRequest {
8262                thread_id: "thread-workspace-entrypoint".to_string(),
8263                message: "investigate voice plan feedback".to_string(),
8264                images: Vec::new(),
8265                provider_override: None,
8266                model_override: None,
8267                reasoning_override: None,
8268                workspace: thread_workspace.display().to_string(),
8269                instructions: crate::instructions::default_instructions(),
8270                developer_context: None,
8271                task_ledger_required: false,
8272                service_tier_override: None,
8273            })
8274            .await
8275            .unwrap();
8276
8277        tokio::time::timeout(std::time::Duration::from_secs(5), async {
8278            loop {
8279                let envelope = rx.recv().await.unwrap();
8280                if envelope.turn_id.as_deref() != Some(&turn_id) {
8281                    continue;
8282                }
8283                match envelope.event {
8284                    RoderEvent::TurnCompleted(_) => break,
8285                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8286                    _ => {}
8287                }
8288            }
8289        })
8290        .await
8291        .unwrap();
8292
8293        let request = captured.lock().unwrap().clone().expect("captured request");
8294        let transcript_text = request
8295            .transcript
8296            .iter()
8297            .map(|item| match item {
8298                TranscriptItem::UserMessage(message) => message.text.as_str(),
8299                _ => "",
8300            })
8301            .collect::<Vec<_>>()
8302            .join("\n");
8303        assert!(transcript_text.contains("src/voice-plan-feedback.ts"));
8304        assert!(!transcript_text.contains("src/sidebar-thread-groups.ts"));
8305
8306        let _ = std::fs::remove_dir_all(process_workspace);
8307        let _ = std::fs::remove_dir_all(thread_workspace);
8308    }
8309
8310    fn runtime_test_workspace(name: &str) -> std::path::PathBuf {
8311        let path =
8312            std::env::temp_dir().join(format!("roder-runtime-{name}-{}", uuid::Uuid::new_v4()));
8313        let _ = std::fs::remove_dir_all(&path);
8314        std::fs::create_dir_all(&path).unwrap();
8315        path
8316    }
8317
8318    #[tokio::test]
8319    async fn model_profile_user_model_knobs_override_profile_defaults() {
8320        let request = captured_profile_request(RuntimeConfig {
8321            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8322            default_model: "gpt-5.5".to_string(),
8323            reasoning: Some(REASONING_HIGH.to_string()),
8324            auto_compact_token_limit: Some(999),
8325            model_edit_tools: std::collections::HashMap::from([(
8326                "gpt-5.5".to_string(),
8327                EDIT_TOOL_PATCH.to_string(),
8328            )]),
8329            model_parallel_tool_calls: std::collections::HashMap::from([(
8330                "gpt-5.5".to_string(),
8331                true,
8332            )]),
8333            model_profiles: std::collections::HashMap::from([(
8334                "gpt-5.5".to_string(),
8335                test_model_profile("gpt-5.5"),
8336            )]),
8337            ..RuntimeConfig::default()
8338        })
8339        .await;
8340
8341        let tool_names = request
8342            .tools
8343            .iter()
8344            .map(|tool| tool.name.as_str())
8345            .collect::<Vec<_>>();
8346        assert!(tool_names.contains(&"apply_patch"));
8347        assert!(!tool_names.contains(&"edit"));
8348        assert_eq!(request.reasoning.level.as_deref(), Some(REASONING_HIGH));
8349        assert_eq!(request.runtime.parallel_tool_calls, Some(true));
8350        assert_eq!(request.runtime.auto_compact_token_limit, Some(999));
8351    }
8352
8353    #[tokio::test]
8354    async fn runtime_tool_allowlist_filters_advertised_tools() {
8355        let request = captured_profile_request(RuntimeConfig {
8356            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8357            default_model: "gpt-5.5".to_string(),
8358            tool_allowlist: vec!["edit".to_string()],
8359            model_profiles: std::collections::HashMap::from([(
8360                "gpt-5.5".to_string(),
8361                test_model_profile("gpt-5.5"),
8362            )]),
8363            ..RuntimeConfig::default()
8364        })
8365        .await;
8366
8367        let tool_names = request
8368            .tools
8369            .iter()
8370            .map(|tool| tool.name.as_str())
8371            .collect::<Vec<_>>();
8372        assert_eq!(tool_names, vec!["edit"]);
8373    }
8374
8375    /// Builds a store-backed runtime turn for a thread created with the given per-thread
8376    /// overrides and returns the captured inference request.
8377    async fn captured_thread_override_request(
8378        runtime: &Arc<Runtime>,
8379        requests: &Arc<StdMutex<Vec<AgentInferenceRequest>>>,
8380        tool_allowlist: Vec<String>,
8381        developer_instructions: Option<String>,
8382        external_tools: Vec<ToolSpec>,
8383    ) -> AgentInferenceRequest {
8384        let thread_id = runtime
8385            .create_thread_with(CreateThreadRequest {
8386                title: Some("Thread overrides".to_string()),
8387                workspace: test_workspace(),
8388                workspace_id: None,
8389                root_id: None,
8390                provider: None,
8391                model: None,
8392                selection_mode: None,
8393                tool_allowlist,
8394                developer_instructions,
8395                external_tools,
8396                runner: None,
8397            })
8398            .await
8399            .unwrap()
8400            .thread_id;
8401        let mut rx = runtime.subscribe_events();
8402        let turn_id = runtime
8403            .start_turn(StartTurnRequest {
8404                thread_id,
8405                message: "hello".to_string(),
8406                images: Vec::new(),
8407                provider_override: None,
8408                model_override: None,
8409                reasoning_override: None,
8410                workspace: test_workspace(),
8411                instructions: crate::default_instructions(),
8412                developer_context: None,
8413                task_ledger_required: false,
8414                service_tier_override: None,
8415            })
8416            .await
8417            .unwrap();
8418        tokio::time::timeout(std::time::Duration::from_secs(5), async {
8419            loop {
8420                let envelope = rx.recv().await.unwrap();
8421                if envelope.turn_id.as_deref() != Some(&turn_id) {
8422                    continue;
8423                }
8424                match envelope.event {
8425                    RoderEvent::TurnCompleted(_) => break,
8426                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8427                    _ => {}
8428                }
8429            }
8430        })
8431        .await
8432        .unwrap();
8433        requests.lock().unwrap().pop().expect("captured request")
8434    }
8435
8436    #[tokio::test]
8437    async fn thread_tool_allowlist_filters_only_that_thread() {
8438        let requests = Arc::new(StdMutex::new(Vec::new()));
8439        let thread_root =
8440            std::env::temp_dir().join(format!("roder-thread-allowlist-{}", uuid::Uuid::new_v4()));
8441        let mut builder = ExtensionRegistryBuilder::new();
8442        builder.inference_engine(Arc::new(SwitchCaptureEngine {
8443            requests: requests.clone(),
8444        }));
8445        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8446            base_path: thread_root.clone(),
8447        }));
8448        builder.tool_contributor(Arc::new(ProfileToolContributor));
8449        let runtime = Arc::new(
8450            Runtime::new(
8451                builder.build().unwrap(),
8452                RuntimeConfig {
8453                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8454                    default_model: "gpt-5.5".to_string(),
8455                    model_profiles: std::collections::HashMap::from([(
8456                        "gpt-5.5".to_string(),
8457                        test_model_profile("gpt-5.5"),
8458                    )]),
8459                    ..RuntimeConfig::default()
8460                },
8461            )
8462            .unwrap(),
8463        );
8464
8465        let allowlisted = captured_thread_override_request(
8466            &runtime,
8467            &requests,
8468            vec!["edit".to_string()],
8469            None,
8470            Vec::new(),
8471        )
8472        .await;
8473        let unrestricted =
8474            captured_thread_override_request(&runtime, &requests, Vec::new(), None, Vec::new())
8475                .await;
8476
8477        let allowlisted_names = allowlisted
8478            .tools
8479            .iter()
8480            .map(|tool| tool.name.as_str())
8481            .collect::<Vec<_>>();
8482        assert_eq!(allowlisted_names, vec!["edit"]);
8483        let unrestricted_names = unrestricted
8484            .tools
8485            .iter()
8486            .map(|tool| tool.name.as_str())
8487            .collect::<Vec<_>>();
8488        assert!(unrestricted_names.contains(&"edit"));
8489        assert!(unrestricted_names.len() > 1);
8490
8491        let _ = std::fs::remove_dir_all(thread_root);
8492    }
8493
8494    /// Builds a store-backed runtime whose `RuntimeConfig.tool_allowlist` is `["edit"]`.
8495    fn runtime_with_edit_allowlist(
8496        requests: &Arc<StdMutex<Vec<AgentInferenceRequest>>>,
8497        thread_root: &std::path::Path,
8498    ) -> Arc<Runtime> {
8499        let mut builder = ExtensionRegistryBuilder::new();
8500        builder.inference_engine(Arc::new(SwitchCaptureEngine {
8501            requests: requests.clone(),
8502        }));
8503        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8504            base_path: thread_root.to_path_buf(),
8505        }));
8506        builder.tool_contributor(Arc::new(ProfileToolContributor));
8507        Arc::new(
8508            Runtime::new(
8509                builder.build().unwrap(),
8510                RuntimeConfig {
8511                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8512                    default_model: "gpt-5.5".to_string(),
8513                    tool_allowlist: vec!["edit".to_string()],
8514                    model_profiles: std::collections::HashMap::from([(
8515                        "gpt-5.5".to_string(),
8516                        test_model_profile("gpt-5.5"),
8517                    )]),
8518                    ..RuntimeConfig::default()
8519                },
8520            )
8521            .unwrap(),
8522        )
8523    }
8524
8525    #[tokio::test]
8526    async fn runtime_and_thread_allowlists_intersect() {
8527        let requests = Arc::new(StdMutex::new(Vec::new()));
8528        let thread_root = std::env::temp_dir().join(format!(
8529            "roder-allowlist-intersect-{}",
8530            uuid::Uuid::new_v4()
8531        ));
8532        let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8533
8534        let request = captured_thread_override_request(
8535            &runtime,
8536            &requests,
8537            vec!["edit".to_string(), "write_file".to_string()],
8538            None,
8539            Vec::new(),
8540        )
8541        .await;
8542
8543        // The thread allowlist must not re-enable tools the runtime allowlist bans.
8544        let names = request
8545            .tools
8546            .iter()
8547            .map(|tool| tool.name.as_str())
8548            .collect::<Vec<_>>();
8549        assert_eq!(names, vec!["edit"]);
8550
8551        let _ = std::fs::remove_dir_all(thread_root);
8552    }
8553
8554    #[tokio::test]
8555    async fn route_tool_call_denies_tools_outside_allowlists() {
8556        let requests = Arc::new(StdMutex::new(Vec::new()));
8557        let thread_root =
8558            std::env::temp_dir().join(format!("roder-allowlist-dispatch-{}", uuid::Uuid::new_v4()));
8559        let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8560        let thread_id = runtime
8561            .create_thread_with(CreateThreadRequest {
8562                title: Some("Dispatch allowlist".to_string()),
8563                workspace: test_workspace(),
8564                workspace_id: None,
8565                root_id: None,
8566                provider: None,
8567                model: None,
8568                selection_mode: None,
8569                tool_allowlist: vec!["edit".to_string(), "write_file".to_string()],
8570                developer_instructions: None,
8571                external_tools: Vec::new(),
8572                runner: None,
8573            })
8574            .await
8575            .unwrap()
8576            .thread_id;
8577
8578        // write_file is registered and on the thread allowlist but banned by the runtime allowlist.
8579        let result = runtime
8580            .route_tool_call(
8581                &thread_id,
8582                &"turn-allowlist-dispatch".to_string(),
8583                roder_api::inference::ToolCallCompleted {
8584                    id: "call-1".to_string(),
8585                    name: "write_file".to_string(),
8586                    arguments: r#"{"path":"a.txt","content":"hi"}"#.to_string(),
8587                },
8588                None,
8589                None,
8590            )
8591            .await
8592            .unwrap();
8593
8594        assert!(result.is_error);
8595        assert!(
8596            result
8597                .result
8598                .contains("not permitted by the tool allowlist"),
8599            "unexpected result: {}",
8600            result.result
8601        );
8602
8603        let _ = std::fs::remove_dir_all(thread_root);
8604    }
8605
8606    #[tokio::test]
8607    async fn route_tool_calls_denies_agent_swarm_mixed_with_other_tools() {
8608        let requests = Arc::new(StdMutex::new(Vec::new()));
8609        let thread_root =
8610            std::env::temp_dir().join(format!("roder-swarm-exclusive-{}", uuid::Uuid::new_v4()));
8611        let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8612
8613        // A response that mixes agent_swarm with another tool is denied wholesale:
8614        // both calls get an error result with retry guidance, and no tool runs.
8615        let results = runtime
8616            .route_tool_calls(
8617                &"thread-swarm".to_string(),
8618                &"turn-swarm".to_string(),
8619                vec![
8620                    roder_api::inference::ToolCallCompleted {
8621                        id: "swarm-1".to_string(),
8622                        name: "agent_swarm".to_string(),
8623                        arguments: "{}".to_string(),
8624                    },
8625                    roder_api::inference::ToolCallCompleted {
8626                        id: "read-1".to_string(),
8627                        name: "read_file".to_string(),
8628                        arguments: "{}".to_string(),
8629                    },
8630                ],
8631                true,
8632                None,
8633                None,
8634            )
8635            .await
8636            .unwrap();
8637
8638        assert_eq!(results.len(), 2, "every tool_call_id must get a response");
8639        assert_eq!(results[0].id, "swarm-1");
8640        assert_eq!(results[1].id, "read-1");
8641        for result in &results {
8642            assert!(result.is_error);
8643            assert!(
8644                result.result.contains("only tool call"),
8645                "unexpected result: {}",
8646                result.result
8647            );
8648        }
8649
8650        let _ = std::fs::remove_dir_all(thread_root);
8651    }
8652
8653    #[tokio::test]
8654    async fn route_tool_calls_emits_agent_swarm_started_event() {
8655        let requests = Arc::new(StdMutex::new(Vec::new()));
8656        let thread_root =
8657            std::env::temp_dir().join(format!("roder-swarm-started-{}", uuid::Uuid::new_v4()));
8658        let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8659        let mut events = runtime.subscribe_events();
8660
8661        // A single agent_swarm call (the valid shape) emits AgentSwarmStarted on
8662        // the event bus before dispatch, with the child count parsed from args.
8663        let _ = runtime
8664            .route_tool_calls(
8665                &"thread-swarm".to_string(),
8666                &"turn-swarm".to_string(),
8667                vec![roder_api::inference::ToolCallCompleted {
8668                    id: "swarm-1".to_string(),
8669                    name: "agent_swarm".to_string(),
8670                    arguments: r#"{"description":"x","prompt_template":"Read {{item}}","items":["a.rs","b.rs"]}"#
8671                        .to_string(),
8672                }],
8673                true,
8674                None,
8675                None,
8676            )
8677            .await
8678            .unwrap();
8679
8680        let mut started_child_count = None;
8681        for _ in 0..16 {
8682            let envelope = tokio::time::timeout(std::time::Duration::from_secs(2), events.recv())
8683                .await
8684                .unwrap()
8685                .unwrap();
8686            if let RoderEvent::AgentSwarmStarted(event) = envelope.event {
8687                started_child_count = Some(event.child_count);
8688                assert_eq!(event.tool_id, "swarm-1");
8689                break;
8690            }
8691        }
8692        assert_eq!(started_child_count, Some(2));
8693
8694        let _ = std::fs::remove_dir_all(thread_root);
8695    }
8696
8697    #[test]
8698    fn agent_swarm_child_count_sums_items_and_resumes() {
8699        assert_eq!(
8700            agent_swarm_child_count(r#"{"description":"x","items":["a","b","c"]}"#),
8701            3
8702        );
8703        assert_eq!(
8704            agent_swarm_child_count(
8705                r#"{"description":"x","items":["a"],"resume_agent_ids":{"id1":"continue"}}"#
8706            ),
8707            2
8708        );
8709        // Malformed input is lenient.
8710        assert_eq!(agent_swarm_child_count("not json"), 0);
8711    }
8712
8713    #[test]
8714    fn parse_swarm_counts_reads_summary_with_omitted_buckets() {
8715        let text = "<agent_swarm_result>\n<summary>completed: 2, failed: 1</summary>\n</agent_swarm_result>";
8716        assert_eq!(parse_swarm_counts(text), Some((2, 1, 0)));
8717        let text = "<agent_swarm_result>\n<summary>completed: 0</summary>\n</agent_swarm_result>";
8718        assert_eq!(parse_swarm_counts(text), Some((0, 0, 0)));
8719        // Not a swarm result.
8720        assert_eq!(parse_swarm_counts("just text"), None);
8721    }
8722
8723    /// Signals when inference starts, then waits for `proceed` and fails the stream.
8724    struct SignalledFailureEngine {
8725        started: tokio::sync::mpsc::UnboundedSender<()>,
8726        proceed: Arc<tokio::sync::Notify>,
8727    }
8728
8729    #[async_trait::async_trait]
8730    impl InferenceEngine for SignalledFailureEngine {
8731        fn id(&self) -> String {
8732            roder_api::catalog::PROVIDER_MOCK.to_string()
8733        }
8734
8735        fn capabilities(&self) -> InferenceCapabilities {
8736            InferenceCapabilities::coding_agent_default()
8737        }
8738
8739        async fn list_models(
8740            &self,
8741            _ctx: InferenceProviderContext<'_>,
8742        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
8743            Ok(roder_api::catalog::models_for_provider(
8744                roder_api::catalog::PROVIDER_MOCK,
8745                true,
8746            ))
8747        }
8748
8749        async fn stream_turn(
8750            &self,
8751            _ctx: InferenceTurnContext<'_>,
8752            _request: AgentInferenceRequest,
8753        ) -> anyhow::Result<InferenceEventStream> {
8754            let _ = self.started.send(());
8755            self.proceed.notified().await;
8756            anyhow::bail!("engine failed mid-turn")
8757        }
8758    }
8759
8760    #[tokio::test]
8761    async fn failed_turn_sweeps_pending_external_tool_calls() {
8762        let (started_tx, mut started_rx) = tokio::sync::mpsc::unbounded_channel();
8763        let proceed = Arc::new(tokio::sync::Notify::new());
8764        let mut builder = ExtensionRegistryBuilder::new();
8765        builder.inference_engine(Arc::new(SignalledFailureEngine {
8766            started: started_tx,
8767            proceed: proceed.clone(),
8768        }));
8769        let runtime =
8770            Arc::new(Runtime::new(builder.build().unwrap(), RuntimeConfig::default()).unwrap());
8771        let mut rx = runtime.subscribe_events();
8772        let turn_id = runtime
8773            .start_turn(StartTurnRequest {
8774                thread_id: "thread-sweep".to_string(),
8775                message: "go".to_string(),
8776                images: Vec::new(),
8777                provider_override: None,
8778                model_override: None,
8779                reasoning_override: None,
8780                workspace: test_workspace(),
8781                instructions: InstructionBundle {
8782                    system: None,
8783                    developer: None,
8784                    developer_context: None,
8785                },
8786                developer_context: None,
8787                task_ledger_required: false,
8788                service_tier_override: None,
8789            })
8790            .await
8791            .unwrap();
8792        tokio::time::timeout(std::time::Duration::from_secs(5), started_rx.recv())
8793            .await
8794            .unwrap()
8795            .unwrap();
8796
8797        let (tx, _pending_rx) = oneshot::channel();
8798        runtime.pending_external_tool_calls.lock().await.insert(
8799            "exttool-sweep-test".to_string(),
8800            PendingExternalToolCall {
8801                thread_id: "thread-sweep".to_string(),
8802                turn_id: turn_id.clone(),
8803                tool_id: "call-1".to_string(),
8804                tool_name: "acme_lookup".to_string(),
8805                tx,
8806            },
8807        );
8808        proceed.notify_one();
8809
8810        let outcome = tokio::time::timeout(std::time::Duration::from_secs(5), async {
8811            loop {
8812                let envelope = rx.recv().await.unwrap();
8813                if let RoderEvent::ExternalToolCallResolved(event) = envelope.event
8814                    && event.request_id == "exttool-sweep-test"
8815                {
8816                    break event.outcome;
8817                }
8818            }
8819        })
8820        .await
8821        .expect("turn failure must resolve pending external tool calls");
8822        assert_eq!(outcome, ExternalToolCallOutcome::Cancelled);
8823        assert!(runtime.pending_external_tool_calls.lock().await.is_empty());
8824    }
8825
8826    #[tokio::test]
8827    async fn thread_developer_instructions_layer_under_harness_prompt() {
8828        let requests = Arc::new(StdMutex::new(Vec::new()));
8829        let thread_root = std::env::temp_dir().join(format!(
8830            "roder-thread-instructions-{}",
8831            uuid::Uuid::new_v4()
8832        ));
8833        let mut builder = ExtensionRegistryBuilder::new();
8834        builder.inference_engine(Arc::new(SwitchCaptureEngine {
8835            requests: requests.clone(),
8836        }));
8837        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8838            base_path: thread_root.clone(),
8839        }));
8840        builder.tool_contributor(Arc::new(ProfileToolContributor));
8841        let runtime =
8842            Arc::new(Runtime::new(builder.build().unwrap(), RuntimeConfig::default()).unwrap());
8843
8844        let request = captured_thread_override_request(
8845            &runtime,
8846            &requests,
8847            Vec::new(),
8848            Some("You are embedded in a host app.".to_string()),
8849            Vec::new(),
8850        )
8851        .await;
8852
8853        let system = request.instructions.system.expect("system instructions");
8854        assert!(system.starts_with("You are Roder"));
8855        let developer = request
8856            .instructions
8857            .developer
8858            .expect("developer instructions");
8859        assert!(developer.starts_with("You are embedded in a host app."));
8860
8861        let plain =
8862            captured_thread_override_request(&runtime, &requests, Vec::new(), None, Vec::new())
8863                .await;
8864        assert_eq!(plain.instructions.developer, None);
8865
8866        let _ = std::fs::remove_dir_all(thread_root);
8867    }
8868
8869    #[tokio::test]
8870    async fn thread_external_tools_are_advertised_and_shadow_builtins() {
8871        let requests = Arc::new(StdMutex::new(Vec::new()));
8872        let thread_root = std::env::temp_dir().join(format!(
8873            "roder-thread-external-tools-{}",
8874            uuid::Uuid::new_v4()
8875        ));
8876        let mut builder = ExtensionRegistryBuilder::new();
8877        builder.inference_engine(Arc::new(SwitchCaptureEngine {
8878            requests: requests.clone(),
8879        }));
8880        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8881            base_path: thread_root.clone(),
8882        }));
8883        builder.tool_contributor(Arc::new(ProfileToolContributor));
8884        let runtime =
8885            Arc::new(Runtime::new(builder.build().unwrap(), RuntimeConfig::default()).unwrap());
8886
8887        let external_tools = vec![
8888            ToolSpec {
8889                name: "acme_lookup".to_string(),
8890                description: "Look up Acme workspace state.".to_string(),
8891                parameters: serde_json::json!({
8892                    "type": "object",
8893                    "properties": { "query": { "type": "string" } },
8894                    "required": ["query"]
8895                }),
8896            },
8897            ToolSpec {
8898                name: "edit".to_string(),
8899                description: "Host-managed edit.".to_string(),
8900                parameters: serde_json::json!({ "type": "object" }),
8901            },
8902        ];
8903        let request =
8904            captured_thread_override_request(&runtime, &requests, Vec::new(), None, external_tools)
8905                .await;
8906
8907        let acme = request
8908            .tools
8909            .iter()
8910            .find(|tool| tool.name == "acme_lookup")
8911            .expect("external tool advertised");
8912        assert_eq!(acme.description, "Look up Acme workspace state.");
8913        assert_eq!(acme.parameters["required"][0], "query");
8914        let edits = request
8915            .tools
8916            .iter()
8917            .filter(|tool| tool.name == "edit")
8918            .collect::<Vec<_>>();
8919        assert_eq!(edits.len(), 1, "external edit shadows the builtin");
8920        assert_eq!(edits[0].description, "Host-managed edit.");
8921
8922        let plain =
8923            captured_thread_override_request(&runtime, &requests, Vec::new(), None, Vec::new())
8924                .await;
8925        assert!(plain.tools.iter().all(|tool| tool.name != "acme_lookup"));
8926        let plain_edit = plain
8927            .tools
8928            .iter()
8929            .find(|tool| tool.name == "edit")
8930            .expect("builtin edit advertised on plain thread");
8931        assert_eq!(plain_edit.description, "edit test tool");
8932
8933        let _ = std::fs::remove_dir_all(thread_root);
8934    }
8935
8936    #[tokio::test]
8937    async fn model_switch_injects_summary_and_records_profile_segments() {
8938        let requests = Arc::new(StdMutex::new(Vec::new()));
8939        let thread_root = std::env::temp_dir().join(format!(
8940            "roder-model-switch-thread-{}",
8941            uuid::Uuid::new_v4()
8942        ));
8943        let mut builder = ExtensionRegistryBuilder::new();
8944        builder.inference_engine(Arc::new(SwitchCaptureEngine {
8945            requests: requests.clone(),
8946        }));
8947        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8948            base_path: thread_root.clone(),
8949        }));
8950        builder.tool_contributor(Arc::new(ProfileToolContributor));
8951        let mut claude_profile = test_model_profile("claude-haiku-4-5-20251001");
8952        claude_profile.provider_family = ProviderFamily::Anthropic;
8953        claude_profile.edit_tool = Some(EDIT_TOOL_EDIT.to_string());
8954        let runtime = Arc::new(
8955            Runtime::new(
8956                builder.build().unwrap(),
8957                RuntimeConfig {
8958                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8959                    default_model: "gpt-5.5".to_string(),
8960                    model_profiles: std::collections::HashMap::from([
8961                        ("gpt-5.5".to_string(), test_model_profile("gpt-5.5")),
8962                        ("claude-haiku-4-5-20251001".to_string(), claude_profile),
8963                    ]),
8964                    ..RuntimeConfig::default()
8965                },
8966            )
8967            .unwrap(),
8968        );
8969        let thread_id = runtime
8970            .create_thread_with(CreateThreadRequest {
8971                title: Some("Model switch".to_string()),
8972                workspace: test_workspace(),
8973                workspace_id: None,
8974                root_id: None,
8975                provider: None,
8976                model: None,
8977                selection_mode: None,
8978                tool_allowlist: Vec::new(),
8979                developer_instructions: None,
8980                external_tools: Vec::new(),
8981                runner: None,
8982            })
8983            .await
8984            .unwrap()
8985            .thread_id;
8986        let mut rx = runtime.subscribe_events();
8987        for (message, model_override) in [
8988            ("first turn", None),
8989            ("second turn", Some("claude-haiku-4-5-20251001".to_string())),
8990        ] {
8991            let turn_id = runtime
8992                .start_turn(StartTurnRequest {
8993                    thread_id: thread_id.clone(),
8994                    message: message.to_string(),
8995                    images: Vec::new(),
8996                    provider_override: None,
8997                    model_override,
8998                    reasoning_override: None,
8999                    workspace: test_workspace(),
9000                    instructions: InstructionBundle::default(),
9001                    developer_context: None,
9002                    task_ledger_required: false,
9003                    service_tier_override: None,
9004                })
9005                .await
9006                .unwrap();
9007            tokio::time::timeout(std::time::Duration::from_secs(5), async {
9008                loop {
9009                    let envelope = rx.recv().await.unwrap();
9010                    if envelope.turn_id.as_deref() != Some(&turn_id) {
9011                        continue;
9012                    }
9013                    match envelope.event {
9014                        RoderEvent::TurnCompleted(_) => break,
9015                        RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9016                        _ => {}
9017                    }
9018                }
9019            })
9020            .await
9021            .unwrap();
9022        }
9023
9024        let captured = requests.lock().unwrap().clone();
9025        assert_eq!(captured.len(), 2);
9026        assert!(captured[1].transcript.iter().any(|item| {
9027            matches!(
9028                item,
9029                TranscriptItem::UserMessage(message)
9030                    if message.text.starts_with(MODEL_SWITCH_SUMMARY_PREFIX)
9031                        && message.text.contains("previous profile mock/gpt-5.5")
9032                        && message.text.contains("Current profile mock/claude-haiku-4-5-20251001")
9033                        && message.text.contains("Available tools now:")
9034            )
9035        }));
9036
9037        let snapshot = runtime
9038            .thread_store
9039            .as_ref()
9040            .unwrap()
9041            .load_thread(&thread_id)
9042            .await
9043            .unwrap()
9044            .unwrap();
9045        let trace_segments = snapshot
9046            .turns
9047            .iter()
9048            .flat_map(|turn| &turn.items)
9049            .filter(|item| {
9050                matches!(
9051                    item,
9052                    TranscriptItem::ProviderMetadata(value)
9053                        if value.get("kind").and_then(serde_json::Value::as_str)
9054                            == Some(MODEL_PROFILE_TRACE_KIND)
9055                            && value.get("segment").and_then(serde_json::Value::as_str)
9056                                == Some("assistant")
9057                )
9058            })
9059            .count();
9060        assert!(trace_segments >= 2);
9061        let _ = std::fs::remove_dir_all(thread_root);
9062    }
9063
9064    struct CountingTaskTool {
9065        calls: Arc<StdMutex<u32>>,
9066    }
9067
9068    #[async_trait::async_trait]
9069    impl ToolExecutor for CountingTaskTool {
9070        fn spec(&self) -> ToolSpec {
9071            ToolSpec {
9072                name: "task".to_string(),
9073                description: "Dispatch a test subagent.".to_string(),
9074                parameters: serde_json::json!({
9075                    "type": "object",
9076                    "properties": {
9077                        "description": { "type": "string" },
9078                        "prompt": { "type": "string" },
9079                        "parent_deadline_seconds": { "type": "integer" }
9080                    },
9081                    "required": ["description", "prompt"],
9082                    "additionalProperties": false
9083                }),
9084            }
9085        }
9086
9087        async fn execute(
9088            &self,
9089            _ctx: ToolExecutionContext,
9090            call: ToolCall,
9091        ) -> anyhow::Result<ToolResult> {
9092            *self.calls.lock().unwrap() += 1;
9093            Ok(ToolResult {
9094                id: call.id,
9095                name: call.name,
9096                text: "started child".to_string(),
9097                data: serde_json::json!({}),
9098                is_error: false,
9099            })
9100        }
9101    }
9102
9103    #[tokio::test]
9104    async fn runtime_profile_reaches_inference_request_and_turn_metadata() {
9105        let captured = Arc::new(StdMutex::new(None));
9106        let mut builder = ExtensionRegistryBuilder::new();
9107        builder.inference_engine(Arc::new(CapturingEngine {
9108            request: captured.clone(),
9109        }));
9110        let runtime = Arc::new(
9111            Runtime::new(
9112                builder.build().unwrap(),
9113                RuntimeConfig {
9114                    runtime_profile: RuntimeProfile::NonInteractive,
9115                    ..RuntimeConfig::default()
9116                },
9117            )
9118            .unwrap(),
9119        );
9120        let mut rx = runtime.subscribe_events();
9121        let turn_id = runtime
9122            .start_turn(StartTurnRequest {
9123                thread_id: "thread-profile".to_string(),
9124                message: "work unattended".to_string(),
9125                images: Vec::new(),
9126                provider_override: None,
9127                model_override: None,
9128                reasoning_override: None,
9129                workspace: test_workspace(),
9130                instructions: InstructionBundle {
9131                    system: None,
9132                    developer: Some("base developer".to_string()),
9133                    developer_context: None,
9134                },
9135                developer_context: None,
9136                task_ledger_required: false,
9137                service_tier_override: None,
9138            })
9139            .await
9140            .unwrap();
9141
9142        let mut observed_profile = None;
9143        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9144            loop {
9145                let envelope = rx.recv().await.unwrap();
9146                if envelope.turn_id.as_deref() != Some(&turn_id) {
9147                    continue;
9148                }
9149                match envelope.event {
9150                    RoderEvent::TurnStarted(event) => {
9151                        observed_profile = Some(event.runtime_profile);
9152                    }
9153                    RoderEvent::TurnCompleted(_) => break,
9154                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9155                    _ => {}
9156                }
9157            }
9158        })
9159        .await
9160        .unwrap();
9161
9162        assert_eq!(observed_profile, Some(RuntimeProfile::NonInteractive));
9163        let request = captured.lock().unwrap().clone().unwrap();
9164        assert_eq!(request.runtime.profile, RuntimeProfile::NonInteractive);
9165        let developer = request.instructions.developer.unwrap();
9166        assert!(developer.contains("base developer"));
9167        assert!(developer.contains("non-interactive profile"));
9168    }
9169
9170    #[tokio::test]
9171    async fn global_policy_mode_changes_do_not_create_runtime_thread_directory() {
9172        let workspace = runtime_test_workspace("global-policy-mode");
9173        let thread_root = workspace.join("threads");
9174        let mut builder = ExtensionRegistryBuilder::new();
9175        builder.inference_engine(Arc::new(FakeInferenceEngine));
9176        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
9177            base_path: thread_root.clone(),
9178        }));
9179        let runtime = Runtime::new(
9180            builder.build().unwrap(),
9181            RuntimeConfig {
9182                workspace: Some(workspace.display().to_string()),
9183                ..Default::default()
9184            },
9185        )
9186        .unwrap();
9187
9188        runtime
9189            .set_policy_mode(PolicyMode::AcceptAll, Some("test".to_string()))
9190            .await
9191            .unwrap();
9192
9193        assert!(!thread_root.join("runtime").exists());
9194        let _ = std::fs::remove_dir_all(workspace);
9195    }
9196
9197    #[tokio::test]
9198    async fn task_ledger_enforcement_injects_eval_reminder_before_work() {
9199        let captured = Arc::new(StdMutex::new(None));
9200        let mut builder = ExtensionRegistryBuilder::new();
9201        builder.inference_engine(Arc::new(CapturingEngine {
9202            request: captured.clone(),
9203        }));
9204        builder.tool_contributor(Arc::new(
9205            roder_ext_task_ledger::TaskLedgerToolContributor::default(),
9206        ));
9207        let runtime = Arc::new(
9208            Runtime::new(
9209                builder.build().unwrap(),
9210                RuntimeConfig {
9211                    runtime_profile: RuntimeProfile::Eval,
9212                    policy_mode: PolicyMode::Bypass,
9213                    agent_swarm_mode: false,
9214                    ultra_mode: false,
9215                    ..RuntimeConfig::default()
9216                },
9217            )
9218            .unwrap(),
9219        );
9220        let mut rx = runtime.subscribe_events();
9221        let turn_id = runtime
9222            .start_turn(StartTurnRequest {
9223                thread_id: "thread-ledger".to_string(),
9224                message: "decomposed work".to_string(),
9225                images: Vec::new(),
9226                provider_override: None,
9227                model_override: None,
9228                reasoning_override: None,
9229                workspace: test_workspace(),
9230                instructions: InstructionBundle::default(),
9231                developer_context: None,
9232                task_ledger_required: true,
9233                service_tier_override: None,
9234            })
9235            .await
9236            .unwrap();
9237
9238        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9239            loop {
9240                let envelope = rx.recv().await.unwrap();
9241                if envelope.turn_id.as_deref() == Some(&turn_id)
9242                    && matches!(envelope.event, RoderEvent::TurnCompleted(_))
9243                {
9244                    break;
9245                }
9246            }
9247        })
9248        .await
9249        .unwrap();
9250
9251        let request = captured.lock().unwrap().clone().unwrap();
9252        let developer = request.instructions.developer.unwrap();
9253        assert!(developer.contains("Task Ledger Required"));
9254        assert!(developer.contains("task_ledger.update"));
9255        let tool_names: Vec<_> = request
9256            .tools
9257            .iter()
9258            .map(|tool| tool.name.as_str())
9259            .collect();
9260        assert!(
9261            tool_names.contains(&TASK_LEDGER_TOOL_NAME),
9262            "tool names: {tool_names:?}"
9263        );
9264        assert_eq!(
9265            request.tool_choice,
9266            ToolChoice::Specific(TASK_LEDGER_TOOL_NAME.to_string())
9267        );
9268        assert_eq!(request.tools.len(), 1);
9269        assert_eq!(request.tools[0].name, TASK_LEDGER_TOOL_NAME);
9270    }
9271
9272    #[tokio::test]
9273    async fn eval_task_ledger_blocks_final_answer_until_open_items_are_completed() {
9274        let requests = Arc::new(StdMutex::new(Vec::new()));
9275        let mut builder = ExtensionRegistryBuilder::new();
9276        builder.inference_engine(Arc::new(TaskLedgerCompletionGateEngine {
9277            calls: StdMutex::new(0),
9278            requests: requests.clone(),
9279        }));
9280        builder.tool_contributor(Arc::new(
9281            roder_ext_task_ledger::TaskLedgerToolContributor::default(),
9282        ));
9283        let runtime = Arc::new(
9284            Runtime::new(
9285                builder.build().unwrap(),
9286                RuntimeConfig {
9287                    runtime_profile: RuntimeProfile::Eval,
9288                    policy_mode: PolicyMode::Bypass,
9289                    agent_swarm_mode: false,
9290                    ultra_mode: false,
9291                    ..RuntimeConfig::default()
9292                },
9293            )
9294            .unwrap(),
9295        );
9296        let mut rx = runtime.subscribe_events();
9297        let turn_id = runtime
9298            .start_turn(StartTurnRequest {
9299                thread_id: "thread-ledger-completion".to_string(),
9300                message: "write the answer file".to_string(),
9301                images: Vec::new(),
9302                provider_override: None,
9303                model_override: None,
9304                reasoning_override: None,
9305                workspace: test_workspace(),
9306                instructions: InstructionBundle::default(),
9307                developer_context: None,
9308                task_ledger_required: true,
9309                service_tier_override: None,
9310            })
9311            .await
9312            .unwrap();
9313
9314        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9315            loop {
9316                let envelope = rx.recv().await.unwrap();
9317                if envelope.turn_id.as_deref() != Some(&turn_id) {
9318                    continue;
9319                }
9320                match envelope.event {
9321                    RoderEvent::TurnCompleted(_) => break,
9322                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9323                    _ => {}
9324                }
9325            }
9326        })
9327        .await
9328        .unwrap();
9329
9330        let requests = requests.lock().unwrap().clone();
9331        assert_eq!(requests.len(), 4);
9332        assert!(requests[2].transcript.iter().any(|item| {
9333            matches!(
9334                item,
9335                TranscriptItem::UserMessage(message)
9336                    if message.text.contains("Task Ledger Completion Required")
9337                        && message.text.contains("Write /app/result.txt")
9338            )
9339        }));
9340        assert!(requests[3].transcript.iter().any(|item| {
9341            matches!(
9342                item,
9343                TranscriptItem::ToolResult(result)
9344                    if result.name.as_deref() == Some(TASK_LEDGER_TOOL_NAME)
9345                        && result.result.contains("Task ledger: 2/2 completed")
9346            )
9347        }));
9348    }
9349
9350    #[tokio::test]
9351    async fn eval_task_ledger_checkpoint_requests_scoreable_file_before_final_reserve() {
9352        let requests = Arc::new(StdMutex::new(Vec::new()));
9353        let mut builder = ExtensionRegistryBuilder::new();
9354        builder.inference_engine(Arc::new(TaskLedgerCompletionGateEngine {
9355            calls: StdMutex::new(0),
9356            requests: requests.clone(),
9357        }));
9358        builder.tool_contributor(Arc::new(
9359            roder_ext_task_ledger::TaskLedgerToolContributor::default(),
9360        ));
9361        let runtime = Arc::new(
9362            Runtime::new(
9363                builder.build().unwrap(),
9364                RuntimeConfig {
9365                    runtime_profile: RuntimeProfile::Eval,
9366                    policy_mode: PolicyMode::Bypass,
9367                    agent_swarm_mode: false,
9368                    ultra_mode: false,
9369                    turn_deadline_seconds: Some(120),
9370                    ..RuntimeConfig::default()
9371                },
9372            )
9373            .unwrap(),
9374        );
9375        let mut rx = runtime.subscribe_events();
9376        let turn_id = runtime
9377            .start_turn(StartTurnRequest {
9378                thread_id: "thread-ledger-checkpoint".to_string(),
9379                message: "write the answer file".to_string(),
9380                images: Vec::new(),
9381                provider_override: None,
9382                model_override: None,
9383                reasoning_override: None,
9384                workspace: test_workspace(),
9385                instructions: InstructionBundle::default(),
9386                developer_context: None,
9387                task_ledger_required: true,
9388                service_tier_override: None,
9389            })
9390            .await
9391            .unwrap();
9392
9393        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9394            loop {
9395                let envelope = rx.recv().await.unwrap();
9396                if envelope.turn_id.as_deref() != Some(&turn_id) {
9397                    continue;
9398                }
9399                match envelope.event {
9400                    RoderEvent::TurnCompleted(_) => break,
9401                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9402                    _ => {}
9403                }
9404            }
9405        })
9406        .await
9407        .unwrap();
9408
9409        let requests = requests.lock().unwrap().clone();
9410        assert!(requests.len() >= 2);
9411        assert!(requests[1].transcript.iter().any(|item| {
9412            matches!(
9413                item,
9414                TranscriptItem::UserMessage(message)
9415                    if message.text.contains("Scoreable Output Checkpoint")
9416                        && message.text.contains("ensure the required output file(s) exist")
9417                        && message.text.contains("Write /app/result.txt")
9418            )
9419        }));
9420    }
9421
9422    #[test]
9423    fn deadline_task_ledger_prompt_preserves_scoreable_work_instruction() {
9424        let prompt = task_ledger_deadline_completion_prompt(
9425            12,
9426            30,
9427            "Task Ledger Completion Required: write /app/result.txt, then call task_ledger.update",
9428        );
9429
9430        assert!(prompt.contains("12 seconds remain"));
9431        assert!(prompt.contains("create or update the required scoreable output files"));
9432        assert!(prompt.contains("write /app/result.txt"));
9433        assert!(prompt.contains(TASK_LEDGER_TOOL_NAME));
9434    }
9435
9436    #[test]
9437    fn scoreable_checkpoint_prompt_preserves_provisional_file_instruction() {
9438        let prompt = task_ledger_scoreable_checkpoint_prompt(
9439            120,
9440            "Task Ledger Completion Required: write /app/result.txt, then call task_ledger.update",
9441        );
9442
9443        assert!(prompt.contains("120 seconds remain"));
9444        assert!(prompt.contains("best evidence-backed answer"));
9445        assert!(prompt.contains("even if provisional"));
9446        assert!(prompt.contains("preserve that candidate"));
9447        assert!(prompt.contains("partial-coverage"));
9448        assert!(prompt.contains("write /app/result.txt"));
9449        assert!(prompt.contains(TASK_LEDGER_TOOL_NAME));
9450    }
9451
9452    #[test]
9453    fn open_task_ledger_moves_inference_timeout_to_scoreable_checkpoint() {
9454        let deadline = Some(OffsetDateTime::now_utc() + Duration::seconds(870));
9455        let transcript = vec![TranscriptItem::ToolResult(ToolResultRecord {
9456            id: "ledger-open".to_string(),
9457            name: Some(TASK_LEDGER_TOOL_NAME.to_string()),
9458            result: "Task ledger: 0/1 completed\n- pending: Write /app/result.txt [write]"
9459                .to_string(),
9460            display_payload: None,
9461            is_error: false,
9462        })];
9463
9464        let (_, action) = inference_timeout_deadline(
9465            deadline,
9466            RuntimeProfile::Eval,
9467            true,
9468            30,
9469            false,
9470            0,
9471            &transcript,
9472        )
9473        .unwrap();
9474
9475        assert_eq!(action, InferenceTimeoutAction::ScoreableCheckpoint);
9476    }
9477
9478    #[tokio::test]
9479    async fn verification_gate_forces_eval_code_changes_through_review() {
9480        let mut builder = ExtensionRegistryBuilder::new();
9481        builder.inference_engine(Arc::new(VerificationGateEngine {
9482            calls: StdMutex::new(0),
9483        }));
9484        builder.tool_contributor(Arc::new(WriteFileContributor));
9485        builder.tool_contributor(Arc::new(
9486            roder_ext_verification::VerificationToolContributor,
9487        ));
9488        let runtime = Arc::new(
9489            Runtime::new(
9490                builder.build().unwrap(),
9491                RuntimeConfig {
9492                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9493                    default_model: "mock".to_string(),
9494                    runtime_profile: RuntimeProfile::Eval,
9495                    policy_mode: PolicyMode::Bypass,
9496                    agent_swarm_mode: false,
9497                    ultra_mode: false,
9498                    ..RuntimeConfig::default()
9499                },
9500            )
9501            .unwrap(),
9502        );
9503        let mut rx = runtime.subscribe_events();
9504        let turn_id = runtime
9505            .start_turn(StartTurnRequest {
9506                thread_id: "thread-verification".to_string(),
9507                message: "write code".to_string(),
9508                images: Vec::new(),
9509                provider_override: None,
9510                model_override: None,
9511                reasoning_override: None,
9512                workspace: test_workspace(),
9513                instructions: InstructionBundle::default(),
9514                developer_context: None,
9515                task_ledger_required: false,
9516                service_tier_override: None,
9517            })
9518            .await
9519            .unwrap();
9520
9521        let mut saw_required = false;
9522        let mut saw_completed = false;
9523        let mut final_text = String::new();
9524        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9525            loop {
9526                let envelope = rx.recv().await.unwrap();
9527                if envelope.turn_id.as_deref() != Some(&turn_id) {
9528                    continue;
9529                }
9530                match envelope.event {
9531                    RoderEvent::VerificationRequired(event) => {
9532                        saw_required = true;
9533                        assert_eq!(event.changed_files, vec!["src/lib.rs"]);
9534                    }
9535                    RoderEvent::VerificationCompleted(event) => {
9536                        saw_completed = true;
9537                        assert!(event.passed);
9538                    }
9539                    RoderEvent::InferenceEventReceived(event) => {
9540                        if let InferenceEvent::MessageDelta(delta) = event.event {
9541                            final_text.push_str(&delta.text);
9542                        }
9543                    }
9544                    RoderEvent::TurnCompleted(_) => break,
9545                    _ => {}
9546                }
9547            }
9548        })
9549        .await
9550        .unwrap();
9551
9552        assert!(saw_required);
9553        assert!(saw_completed);
9554        assert!(final_text.contains("verified final"));
9555    }
9556
9557    #[tokio::test]
9558    async fn speed_policy_changes_reasoning_across_eval_model_calls_without_model_switch() {
9559        let requests = Arc::new(StdMutex::new(Vec::new()));
9560        let mut builder = ExtensionRegistryBuilder::new();
9561        builder.inference_engine(Arc::new(SpeedPolicyEngine {
9562            calls: StdMutex::new(0),
9563            requests: requests.clone(),
9564        }));
9565        builder.tool_contributor(Arc::new(WriteFileContributor));
9566        builder.tool_contributor(Arc::new(
9567            roder_ext_verification::VerificationToolContributor,
9568        ));
9569        let runtime = Arc::new(
9570            Runtime::new(
9571                builder.build().unwrap(),
9572                RuntimeConfig {
9573                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9574                    default_model: "gpt-5.5".to_string(),
9575                    runtime_profile: RuntimeProfile::Eval,
9576                    policy_mode: PolicyMode::Bypass,
9577                    agent_swarm_mode: false,
9578                    ultra_mode: false,
9579                    ..RuntimeConfig::default()
9580                },
9581            )
9582            .unwrap(),
9583        );
9584        let mut rx = runtime.subscribe_events();
9585        let turn_id = runtime
9586            .start_turn(StartTurnRequest {
9587                thread_id: "thread-speed-policy".to_string(),
9588                message: "write code".to_string(),
9589                images: Vec::new(),
9590                provider_override: None,
9591                model_override: None,
9592                reasoning_override: None,
9593                workspace: test_workspace(),
9594                instructions: InstructionBundle::default(),
9595                developer_context: None,
9596                task_ledger_required: false,
9597                service_tier_override: None,
9598            })
9599            .await
9600            .unwrap();
9601
9602        let mut saw_speed_policy_event = false;
9603        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9604            loop {
9605                let envelope = rx.recv().await.unwrap();
9606                if envelope.turn_id.as_deref() != Some(&turn_id) {
9607                    continue;
9608                }
9609                match envelope.event {
9610                    RoderEvent::InferenceStarted(event) => {
9611                        if event.speed_policy.is_some() {
9612                            saw_speed_policy_event = true;
9613                        }
9614                    }
9615                    RoderEvent::TurnCompleted(_) => break,
9616                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9617                    _ => {}
9618                }
9619            }
9620        })
9621        .await
9622        .unwrap();
9623
9624        let requests = requests.lock().unwrap().clone();
9625        assert!(saw_speed_policy_event);
9626        assert!(requests.len() >= 4);
9627        assert!(requests.iter().all(|request| {
9628            request.model.provider == roder_api::catalog::PROVIDER_MOCK
9629                && request.model.model == "gpt-5.5"
9630        }));
9631        assert_eq!(
9632            requests[0].runtime.speed_policy.as_ref().map(|d| d.phase),
9633            Some(roder_api::inference::SpeedPolicyPhase::Orientation)
9634        );
9635        assert_eq!(requests[0].reasoning.level.as_deref(), Some(REASONING_HIGH));
9636        assert_eq!(
9637            requests[1].runtime.speed_policy.as_ref().map(|d| d.phase),
9638            Some(roder_api::inference::SpeedPolicyPhase::Execution)
9639        );
9640        assert_eq!(requests[1].reasoning.level.as_deref(), Some(REASONING_LOW));
9641        assert_eq!(
9642            requests[2].runtime.speed_policy.as_ref().map(|d| d.phase),
9643            Some(roder_api::inference::SpeedPolicyPhase::Verification)
9644        );
9645        assert_eq!(requests[2].reasoning.level.as_deref(), Some(REASONING_HIGH));
9646        assert_eq!(
9647            requests[2]
9648                .metadata
9649                .pointer("/speedPolicy/phase")
9650                .and_then(serde_json::Value::as_str),
9651            Some("verification")
9652        );
9653    }
9654
9655    #[tokio::test]
9656    async fn deadline_turn_timeout_emits_partial_result_and_clears_active_turn() {
9657        let mut builder = ExtensionRegistryBuilder::new();
9658        builder.inference_engine(Arc::new(DeadlineEngine));
9659        let runtime = Arc::new(
9660            Runtime::new(
9661                builder.build().unwrap(),
9662                RuntimeConfig {
9663                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9664                    default_model: "mock".to_string(),
9665                    runtime_profile: RuntimeProfile::Eval,
9666                    turn_deadline_seconds: Some(1),
9667                    ..RuntimeConfig::default()
9668                },
9669            )
9670            .unwrap(),
9671        );
9672        let mut rx = runtime.subscribe_events();
9673        let turn_id = runtime
9674            .start_turn(StartTurnRequest {
9675                thread_id: "thread-deadline".to_string(),
9676                message: "slow work".to_string(),
9677                images: Vec::new(),
9678                provider_override: None,
9679                model_override: None,
9680                reasoning_override: None,
9681                workspace: test_workspace(),
9682                instructions: InstructionBundle::default(),
9683                developer_context: None,
9684                task_ledger_required: false,
9685                service_tier_override: None,
9686            })
9687            .await
9688            .unwrap();
9689
9690        let mut saw_partial = false;
9691        let mut saw_deadline = false;
9692        let mut failed_kind = None;
9693        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9694            loop {
9695                let envelope = rx.recv().await.unwrap();
9696                if envelope.turn_id.as_deref() != Some(&turn_id) {
9697                    continue;
9698                }
9699                match envelope.event {
9700                    RoderEvent::TurnPartialResult(event) => {
9701                        saw_partial = event.summary.contains("partial turn state");
9702                    }
9703                    RoderEvent::TurnDeadlineExceeded(event) => {
9704                        saw_deadline = event.partial_result.contains("transcript items");
9705                    }
9706                    RoderEvent::TurnFailed(event) => {
9707                        failed_kind = event.error_kind;
9708                        break;
9709                    }
9710                    _ => {}
9711                }
9712            }
9713        })
9714        .await
9715        .unwrap();
9716
9717        for _ in 0..20 {
9718            if !runtime.active_turns.read().await.contains_key(&turn_id) {
9719                break;
9720            }
9721            tokio::time::sleep(std::time::Duration::from_millis(10)).await;
9722        }
9723        assert!(saw_partial);
9724        assert!(saw_deadline);
9725        assert_eq!(failed_kind.as_deref(), Some("deadline_timeout"));
9726        assert!(!runtime.active_turns.read().await.contains_key(&turn_id));
9727    }
9728
9729    #[tokio::test]
9730    async fn deadline_skips_subagent_task_when_remaining_budget_is_too_low() {
9731        let calls = Arc::new(StdMutex::new(0));
9732        let mut builder = ExtensionRegistryBuilder::new();
9733        builder.inference_engine(Arc::new(CapturingEngine {
9734            request: Arc::new(StdMutex::new(None)),
9735        }));
9736        let task_tool = Arc::new(CountingTaskTool {
9737            calls: calls.clone(),
9738        });
9739        builder.tool_contributor(Arc::new(TestToolContributor { tool: task_tool }));
9740        let runtime = Arc::new(
9741            Runtime::new(
9742                builder.build().unwrap(),
9743                RuntimeConfig {
9744                    policy_mode: PolicyMode::Bypass,
9745                    agent_swarm_mode: false,
9746                    ultra_mode: false,
9747                    ..RuntimeConfig::default()
9748                },
9749            )
9750            .unwrap(),
9751        );
9752
9753        let result = runtime
9754            .route_tool_call(
9755                &"thread-deadline-task".to_string(),
9756                &"turn-deadline-task".to_string(),
9757                ToolCallCompleted {
9758                    id: "task-1".to_string(),
9759                    name: "task".to_string(),
9760                    arguments: serde_json::json!({
9761                        "description": "inspect",
9762                        "prompt": "read"
9763                    })
9764                    .to_string(),
9765                },
9766                None,
9767                Some(OffsetDateTime::now_utc() + Duration::seconds(1)),
9768            )
9769            .await
9770            .unwrap();
9771
9772        assert!(result.is_error);
9773        assert!(result.result.contains("deadline policy skipped"));
9774        assert_eq!(*calls.lock().unwrap(), 0);
9775    }
9776
9777    struct TestToolContributor {
9778        tool: Arc<dyn ToolExecutor>,
9779    }
9780
9781    impl ToolContributor for TestToolContributor {
9782        fn id(&self) -> String {
9783            "test-tool".to_string()
9784        }
9785
9786        fn contribute(&self, registry: &mut ToolRegistry) -> anyhow::Result<()> {
9787            registry.register(self.tool.clone())
9788        }
9789    }
9790
9791    #[tokio::test]
9792    async fn agent_swarm_mode_override_is_per_thread() {
9793        let runtime = Runtime::fake().unwrap();
9794        let trigger = roder_api::subagents::AgentSwarmModeTrigger::Manual;
9795
9796        // Off everywhere by default.
9797        assert!(
9798            !runtime
9799                .effective_agent_swarm_mode_for_thread("thread-a")
9800                .await
9801        );
9802        assert!(
9803            !runtime
9804                .effective_agent_swarm_mode_for_thread("thread-b")
9805                .await
9806        );
9807
9808        // Enabling on thread-a does not leak into thread-b.
9809        assert!(
9810            runtime
9811                .set_agent_swarm_mode_for_thread("thread-a", true, trigger)
9812                .await
9813        );
9814        assert!(
9815            runtime
9816                .effective_agent_swarm_mode_for_thread("thread-a")
9817                .await
9818        );
9819        assert!(
9820            !runtime
9821                .effective_agent_swarm_mode_for_thread("thread-b")
9822                .await
9823        );
9824
9825        // A per-thread `off` override wins over a runtime-global `on` default.
9826        runtime.set_agent_swarm_mode(true, trigger).await.unwrap();
9827        runtime
9828            .set_agent_swarm_mode_for_thread("thread-b", false, trigger)
9829            .await;
9830        assert!(
9831            !runtime
9832                .effective_agent_swarm_mode_for_thread("thread-b")
9833                .await,
9834            "explicit per-thread off overrides the global on default"
9835        );
9836        // A thread with no override still follows the global default.
9837        assert!(
9838            runtime
9839                .effective_agent_swarm_mode_for_thread("thread-c")
9840                .await,
9841            "threads without an override follow the runtime-global default"
9842        );
9843    }
9844
9845    #[tokio::test]
9846    async fn set_agent_swarm_mode_for_thread_emits_event_with_real_thread_id() {
9847        let runtime = Runtime::fake().unwrap();
9848        let mut events = runtime.subscribe_events();
9849        runtime
9850            .set_agent_swarm_mode_for_thread(
9851                "thread-xyz",
9852                true,
9853                roder_api::subagents::AgentSwarmModeTrigger::Task,
9854            )
9855            .await;
9856        let mut saw = false;
9857        for _ in 0..8 {
9858            let envelope = tokio::time::timeout(std::time::Duration::from_secs(2), events.recv())
9859                .await
9860                .unwrap()
9861                .unwrap();
9862            if let RoderEvent::AgentSwarmModeChanged(event) = envelope.event {
9863                assert_eq!(event.thread_id, "thread-xyz");
9864                assert!(event.enabled);
9865                assert_eq!(
9866                    event.trigger,
9867                    roder_api::subagents::AgentSwarmModeTrigger::Task
9868                );
9869                saw = true;
9870                break;
9871            }
9872        }
9873        assert!(
9874            saw,
9875            "expected an AgentSwarmModeChanged event for the thread"
9876        );
9877    }
9878
9879    #[tokio::test]
9880    async fn ultra_mode_override_is_per_thread() {
9881        let runtime = Runtime::fake().unwrap();
9882        let trigger = roder_api::subagents::UltraModeTrigger::Manual;
9883
9884        assert!(!runtime.effective_ultra_mode_for_thread("thread-a").await);
9885        assert!(!runtime.effective_ultra_mode_for_thread("thread-b").await);
9886
9887        assert!(
9888            runtime
9889                .set_ultra_mode_for_thread("thread-a", true, trigger)
9890                .await
9891        );
9892        assert!(runtime.effective_ultra_mode_for_thread("thread-a").await);
9893        assert!(!runtime.effective_ultra_mode_for_thread("thread-b").await);
9894
9895        runtime.set_ultra_mode(true, trigger).await.unwrap();
9896        runtime
9897            .set_ultra_mode_for_thread("thread-b", false, trigger)
9898            .await;
9899        assert!(
9900            !runtime.effective_ultra_mode_for_thread("thread-b").await,
9901            "explicit per-thread off overrides the global on default"
9902        );
9903        assert!(
9904            runtime.effective_ultra_mode_for_thread("thread-c").await,
9905            "threads without an override follow the runtime-global default"
9906        );
9907    }
9908
9909    #[tokio::test]
9910    async fn set_ultra_mode_for_thread_emits_event_with_real_thread_id() {
9911        let runtime = Runtime::fake().unwrap();
9912        let mut events = runtime.subscribe_events();
9913        runtime
9914            .set_ultra_mode_for_thread(
9915                "thread-ultra",
9916                true,
9917                roder_api::subagents::UltraModeTrigger::Task,
9918            )
9919            .await;
9920        let mut saw = false;
9921        for _ in 0..8 {
9922            let envelope = tokio::time::timeout(std::time::Duration::from_secs(2), events.recv())
9923                .await
9924                .unwrap()
9925                .unwrap();
9926            if let RoderEvent::UltraModeChanged(event) = envelope.event {
9927                assert_eq!(event.thread_id, "thread-ultra");
9928                assert!(event.enabled);
9929                assert_eq!(event.trigger, roder_api::subagents::UltraModeTrigger::Task);
9930                saw = true;
9931                break;
9932            }
9933        }
9934        assert!(saw, "expected an UltraModeChanged event for the thread");
9935    }
9936}
9937
9938#[cfg(test)]
9939#[path = "runtime/mailbox_test_helpers.rs"]
9940mod mailbox_test_helpers;