Skip to main content

roder_core/
runtime.rs

1#[path = "runtime/sampling.rs"]
2pub(crate) mod sampling;
3#[path = "tool_advertisement.rs"]
4mod tool_advertisement;
5use sampling::{
6    SamplingPreempted, SamplingRetry, completed_call_replayed, preemptible, wait_for_steer,
7};
8#[path = "runtime/sampling_output.rs"]
9mod sampling_output;
10use sampling_output::{OutputContext, SamplingOutput};
11mod compaction_template;
12#[path = "runtime/eager_tools.rs"]
13mod eager_tools;
14mod execution_receipts;
15mod owner_handoff;
16mod thread_admission;
17#[path = "runtime/tool_approvals.rs"]
18mod tool_approvals;
19#[path = "runtime/turn_start.rs"]
20mod turn_start;
21use eager_tools::EagerTools;
22#[path = "runtime/patch_progress.rs"]
23mod patch_progress;
24use patch_progress::PatchProgressTracker;
25
26use std::collections::{HashMap, HashSet};
27use std::path::PathBuf;
28use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
29use std::sync::{Arc, Weak};
30
31use anyhow::Context;
32use futures::StreamExt;
33use futures::future::{AbortHandle, Abortable, BoxFuture, try_join_all};
34use roder_api::catalog::{
35    EDIT_TOOL_EDIT, EDIT_TOOL_PATCH, PROVIDER_GEMINI, REASONING_NONE, REASONING_ULTRA,
36    built_in_model_profile, built_in_model_profile_for_provider, lookup_model,
37    model_supports_reasoning_effort,
38};
39use roder_api::context::PolicyGate;
40use roder_api::events::*;
41use roder_api::extension::ExtensionRegistry;
42use roder_api::inference::{
43    AgentInferenceRequest, HostedWebSearchConfig, HostedWebSearchMode, InferenceEngine,
44    InferenceEvent, InferenceTurnContext, InstructionBundle, ModelHarnessProfile,
45    ModelSchemaPolicy, ModelSelection, OutputConfig, ProviderTurnCleanup, ReasoningConfig,
46    RuntimeHints, RuntimeProfile, TokenUsage, ToolCallCompleted, ToolSearchConfig,
47    ToolSearchConfigOverlay, finish_reason_from_stop_reason,
48};
49use roder_api::inference_routing::{InferenceRoutingOutcome, ModelSelectionMode};
50use roder_api::lifecycle::{
51    LifecycleMetricsSnapshot, TurnCleanupOwnership, TurnCleanupState, TurnLifecycleReason,
52    TurnLifecycleRecord, TurnLifecycleSnapshot, TurnLifecycleState, turn_lifecycle_snapshot,
53};
54use roder_api::policy_mode::{PolicyDecision, PolicyMode};
55use roder_api::reliability::{
56    ReliabilityContext, ReliabilityDetails, ReliabilityErrorClass, ReliabilityLimitDecision,
57    ReliabilityLimitRecorded, ReliabilityRequestPolicy, ReliabilityRetryDecision,
58    ReliabilityRetryRecorded, provider_retry_delay_ms,
59};
60use roder_api::remote_runner::{
61    RemoteRunnerProvider, RemoteRunnerSession, RemoteWorkspace, RunnerDestination,
62    RunnerSessionState, ThreadRunnerBinding,
63};
64use roder_api::subagents::SubagentDefinition;
65use roder_api::teams::{
66    TeamId, TeamMailboxMessage, TeamMailboxMessageKind, TeamMemberDescriptor, TeamMemberRole,
67    TeamMemberStatus,
68};
69use roder_api::thread::{
70    ThreadItemEvent, ThreadItemEventKind, ThreadMetadata, ThreadSnapshot, ThreadStore,
71    ThreadUsageMetadata, is_synthetic_event_thread_id, validate_thread_workspace,
72};
73use roder_api::tools::{ToolCall, ToolChoice, ToolExecutionContext, ToolRegistry, ToolResult};
74use roder_api::transcript::{
75    AssistantMessage, ErrorRecord, InputImage, ReasoningSummary, ToolCallRecord, ToolResultRecord,
76    TranscriptItem, UserMessage,
77};
78use roder_sandbox::ScopedFilesystem;
79use roder_sandbox::process::LocalProcessRunner;
80use roder_skills::{SkillRegistry, SkillRegistryOptions};
81use time::{Duration, OffsetDateTime};
82use tokio::sync::{Mutex, Notify, RwLock, oneshot};
83
84mod codex_v2;
85
86use crate::artifacts::{
87    ContextArtifactStore as FilesystemContextArtifactStore, default_context_artifact_dir,
88};
89use crate::browser_routing::apply_browser_routing;
90use crate::bus::EventBus;
91use crate::dynamic_workflows::{
92    DynamicWorkflowEffortProfile, RuntimeDynamicWorkflowConfig, WorkflowTriggerDecision,
93    classify_workflow_trigger, ultracode_reasoning_level_for_model,
94};
95use crate::fake_provider::FakeInferenceEngine;
96use crate::goals::RuntimeGoalController;
97use crate::inference_routing::{
98    InferenceRoutingRequest, RuntimeInferenceRouterConfig, collect_inference_routing_candidates,
99    route_inference_selection, transcript_failure_count_since,
100};
101use crate::instructions::{
102    apply_agent_swarm_mode, apply_codex_multi_agent_mode, apply_model_instruction_overlay,
103    apply_parallel_web_tools, apply_plan_mode, apply_runtime_profile, apply_task_ledger_required,
104    apply_thread_developer_instructions, apply_turn_developer_context,
105};
106use crate::policy_gate::DefaultPolicyGate;
107use crate::reliability::{
108    ReliabilityLimitHit, RuntimeReliabilityConfig, TurnReliabilityState,
109    provider_stream_retry_cause,
110};
111pub use crate::speed_policy::RuntimeSpeedPolicyConfig;
112use crate::speed_policy::{SpeedPolicyState, reasoning_from_decision};
113use crate::subagent_traces::{RuntimeAgentSwarmProgressSink, RuntimeSubagentTraceSink};
114use crate::teams::{TeamManager, TeamMemberStartRequest, TeamStartRequest, TeamState};
115use crate::thread_item_cache::{ThreadItemCache, ThreadItemCacheEntry};
116use crate::tool_result_images::ToolImages;
117use crate::verification_gate::VerificationGateState;
118
119const MAX_TOOL_ROUNDS_PER_TURN: usize = 1024;
120/// Injected when `continue_on_failure_limit` turns a consecutive-tool-failure
121/// limit into a continuation, or as the empty-tool-call persistence nudge, so
122/// the model keeps working instead of ending the turn early.
123const RELIABILITY_CONTINUATION_PROMPT: &str = "Verify the task is fully complete. If any part remains unfinished or unverified, keep working using the available tools — try an alternative approach if one is failing. Only stop once the solution is complete and verified; if it is already complete, restate your final answer.";
124/// Capacity of the runtime event broadcast ring buffer. The TUI drains this on
125/// its render loop, which during an active turn only wakes at the ~6 FPS status
126/// animation cadence (backend events do not wake the input poll), so up to
127/// ~166ms of events buffer between drains. A reasoning-high provider that runs
128/// its whole tool loop inside one turn (e.g. claude-code) streams a firehose of
129/// `InferenceEventReceived` deltas plus per-step tool/thinking events; the old
130/// 1024-slot buffer overflowed in those windows and silently dropped tool and
131/// thinking rows from the live view. Sized for generous headroom across bursts
132/// and brief render stalls.
133const EVENT_BUS_CAPACITY: usize = 16_384;
134
135/// Turn id reported to session-scoped local hooks. `SessionStart` and
136/// `SessionEnd` bracket the session rather than any one turn, but the hook
137/// payload carries a turn id, so they share this synthetic value.
138const SESSION_HOOK_TURN_ID: &str = "session";
139pub(crate) const FINAL_ANSWER_PHASE: &str = "final_answer";
140pub(crate) const TASK_LEDGER_TOOL_NAME: &str = "task_ledger.update";
141const TASK_LEDGER_COMPLETION_REMINDER_LIMIT: u8 = 2;
142const TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS: u64 = 180;
143const TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT: u8 = 1;
144pub(crate) const MIN_CHILD_DEADLINE_SECONDS: u64 = 2;
145const MODEL_PROFILE_TRACE_KIND: &str = "model_profile_segment";
146const MODEL_SWITCH_SUMMARY_PREFIX: &str = "Model switch summary:";
147const PROVIDER_CLEANUP_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5);
148
149#[derive(Clone, Copy, Debug, Eq, PartialEq)]
150enum InferenceTimeoutAction {
151    ScoreableCheckpoint,
152    Finalization,
153}
154
155#[derive(Debug, Clone)]
156pub struct RuntimeConfig {
157    pub default_provider: String,
158    pub default_model: String,
159    pub reasoning: Option<String>,
160    pub auto_compact_token_limit: Option<u32>,
161    pub file_backed_dynamic_context: bool,
162    pub hosted_web_search: HostedWebSearchConfig,
163    pub tool_search: ToolSearchConfig,
164    pub provider_tool_search: HashMap<String, ToolSearchConfigOverlay>,
165    pub model_tool_search: HashMap<String, ToolSearchConfigOverlay>,
166    pub model_edit_tools: HashMap<String, String>,
167    pub model_parallel_tool_calls: HashMap<String, bool>,
168    pub model_profiles: HashMap<String, ModelHarnessProfile>,
169    pub tool_allowlist: Vec<String>,
170    /// Seconds a host-executed external tool call may stay unresolved before it fails with a timeout error.
171    pub external_tool_timeout_seconds: u64,
172    pub command_shell: String,
173    pub workspace: Option<String>,
174    pub policy_mode: PolicyMode,
175    /// Whether agent-swarm mode is active (roadmap 104). When on, the runtime
176    /// injects the swarm reminder into each turn's developer instructions so any
177    /// client benefits, not just the TUI.
178    pub agent_swarm_mode: bool,
179    /// Whether ultra mode is active. When on, the runtime injects proactive
180    /// multi-agent delegation policy for any model (not only Codex Sol/Terra
181    /// Ultra effort). Selecting Ultra reasoning on Sol/Terra still enables
182    /// proactive multi-agent for that model without requiring this flag.
183    pub ultra_mode: bool,
184    pub runtime_profile: RuntimeProfile,
185    pub inference_router: RuntimeInferenceRouterConfig,
186    pub speed_policy: RuntimeSpeedPolicyConfig,
187    pub dynamic_workflows: RuntimeDynamicWorkflowConfig,
188    pub reliability: RuntimeReliabilityConfig,
189    /// Positive wall-clock limit for a turn in any runtime profile. `None` or zero is unbounded.
190    pub turn_deadline_seconds: Option<u64>,
191    pub remote_runner_destination: Option<RunnerDestination>,
192    pub team_data_dir: Option<PathBuf>,
193    pub roadmap_data_dir: Option<PathBuf>,
194    pub media_generation: crate::media_generation::RuntimeMediaGenerationConfig,
195    /// `[review]` settings: the review sub-turn and its publishers.
196    pub review: crate::review::RuntimeReviewConfig,
197}
198
199impl Default for RuntimeConfig {
200    fn default() -> Self {
201        Self {
202            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
203            default_model: "mock".to_string(),
204            reasoning: None,
205            auto_compact_token_limit: None,
206            file_backed_dynamic_context: true,
207            hosted_web_search: HostedWebSearchConfig::cached(),
208            tool_search: ToolSearchConfig::default(),
209            provider_tool_search: HashMap::new(),
210            model_tool_search: HashMap::new(),
211            model_edit_tools: HashMap::new(),
212            model_parallel_tool_calls: HashMap::new(),
213            model_profiles: HashMap::new(),
214            tool_allowlist: Vec::new(),
215            external_tool_timeout_seconds: DEFAULT_EXTERNAL_TOOL_TIMEOUT_SECONDS,
216            command_shell: roder_api::command_shell::default_command_shell(),
217            workspace: None,
218            policy_mode: PolicyMode::Default,
219            agent_swarm_mode: false,
220            ultra_mode: false,
221            runtime_profile: RuntimeProfile::Interactive,
222            inference_router: RuntimeInferenceRouterConfig::default(),
223            speed_policy: RuntimeSpeedPolicyConfig::default(),
224            dynamic_workflows: RuntimeDynamicWorkflowConfig::default(),
225            reliability: RuntimeReliabilityConfig::default(),
226            turn_deadline_seconds: None,
227            remote_runner_destination: None,
228            team_data_dir: None,
229            roadmap_data_dir: None,
230            media_generation: crate::media_generation::RuntimeMediaGenerationConfig::default(),
231            review: crate::review::RuntimeReviewConfig::default(),
232        }
233    }
234}
235
236#[derive(Debug, Clone)]
237pub struct StartTurnRequest {
238    pub thread_id: ThreadId,
239    pub message: String,
240    pub images: Vec<InputImage>,
241    pub provider_override: Option<String>,
242    pub model_override: Option<String>,
243    pub reasoning_override: Option<String>,
244    pub workspace: String,
245    pub instructions: InstructionBundle,
246    /**
247     * Per-turn developer-authority context for this turn's InstructionBundle.
248     * Applies to every inference round of the turn, is never written to
249     * thread state. Automatic goal continuations retain it; later explicit
250     * turns receive only the context supplied by their host.
251     */
252    pub developer_context: Option<String>,
253    pub task_ledger_required: bool,
254    /**
255     * Per-turn provider service tier (OpenAI `service_tier`, e.g.
256     * `"priority"` for Fast mode). Carried to every inference round of the
257     * turn as `RuntimeHints::service_tier`; `None` keeps the provider
258     * default. Providers without a tier concept ignore it.
259     */
260    pub service_tier_override: Option<String>,
261}
262
263#[derive(Debug, Clone)]
264pub struct CreateThreadRequest {
265    pub title: Option<String>,
266    pub workspace: String,
267    pub workspace_id: Option<String>,
268    pub root_id: Option<String>,
269    pub provider: Option<String>,
270    pub model: Option<String>,
271    pub selection_mode: Option<ModelSelectionMode>,
272    /// Per-thread tool filter applied on top of the runtime allowlist. Empty = no filtering.
273    pub tool_allowlist: Vec<String>,
274    /// Host-supplied instructions added to the developer slot of every turn's inference request.
275    pub developer_instructions: Option<String>,
276    /// Host-executed tool specs advertised to the model on every turn of this thread.
277    pub external_tools: Vec<roder_api::tools::ToolSpec>,
278    /// Explicit remote-runner binding for the thread's native coding tools.
279    pub runner: Option<ThreadRunnerSelection>,
280}
281
282/**
283 * Thread-level remote-runner selection. The destination config is persisted
284 * with the thread, so secrets must reach the provider through its
285 * environment, not this config.
286 */
287#[derive(Debug, Clone)]
288pub struct ThreadRunnerSelection {
289    pub provider_id: String,
290    pub config: serde_json::Value,
291    /// Absolute path on the runner used as the thread's coding-tool workspace root.
292    pub workspace: String,
293    /**
294     * Extra absolute runner paths file reads may resolve under, beyond
295     * `workspace`. Writes and the working directory stay confined to
296     * `workspace`.
297     */
298    pub read_roots: Vec<String>,
299}
300
301#[derive(Debug, Clone, PartialEq, Eq)]
302pub struct PendingPlanExit {
303    pub thread_id: ThreadId,
304    pub turn_id: TurnId,
305    pub request_id: String,
306    pub target_mode: PolicyMode,
307    pub plan_summary: Option<String>,
308    pub next_steps: Vec<String>,
309    pub requested_at: OffsetDateTime,
310    pub expires_at: Option<OffsetDateTime>,
311}
312
313pub(crate) struct PendingToolApproval {
314    pub(crate) thread_id: ThreadId,
315    pub(crate) turn_id: TurnId,
316    pub(crate) tool_id: String,
317    pub(crate) tool_name: String,
318    pub(crate) call: roder_api::tools::ToolCall,
319    pub(crate) context: Option<ToolExecutionContext>,
320    pub(crate) tx: oneshot::Sender<bool>,
321}
322
323pub(crate) struct PendingUserInput {
324    pub(crate) thread_id: ThreadId,
325    pub(crate) turn_id: TurnId,
326    pub(crate) tx: oneshot::Sender<serde_json::Value>,
327}
328
329/// Host answer to an external tool call delivered via `tools/resolve`.
330#[derive(Debug, Clone, PartialEq, Eq)]
331pub struct ExternalToolResolution {
332    pub output: String,
333    pub is_error: bool,
334}
335
336pub(crate) struct PendingExternalToolCall {
337    pub(crate) thread_id: ThreadId,
338    pub(crate) turn_id: TurnId,
339    pub(crate) tool_id: String,
340    pub(crate) tool_name: String,
341    pub(crate) tx: oneshot::Sender<ExternalToolResolution>,
342}
343
344#[derive(Clone)]
345struct ActiveTurnHandle {
346    thread_id: ThreadId,
347    abort: AbortHandle,
348    steers: Arc<Mutex<Vec<QueuedTurnSteer>>>,
349    steer_changed: tokio::sync::watch::Sender<u64>,
350    drain: Arc<TurnDrainHandle>,
351}
352
353/// Tracks a task after it has been interrupted and removed from the interactive
354/// active-turn map. This lets users start a follow-up turn immediately while a
355/// bounded runtime shutdown can still await the original task's cleanup.
356struct TurnDrainHandle {
357    thread_id: ThreadId,
358    interrupt_requested: AtomicBool,
359    interrupt_reason: Mutex<Option<TurnLifecycleReason>>,
360    completed: AtomicBool,
361    completed_notify: Notify,
362}
363
364/// Result of a bounded runtime drain. `Clean` proves the runtime's own turn
365/// tasks have reached their cleanup paths; provider-specific child-process
366/// reaping remains an explicit provider capability.
367#[derive(Debug, Clone, PartialEq, Eq)]
368pub enum RuntimeDrainOutcome {
369    Clean {
370        interrupted_turn_ids: Vec<TurnId>,
371    },
372    DeadlineExceeded {
373        interrupted_turn_ids: Vec<TurnId>,
374        remaining_turn_ids: Vec<TurnId>,
375    },
376    /// At least one lifecycle transition initiated by this drain could not be
377    /// durably appended. The runtime still made its best cleanup attempt, but
378    /// callers must not claim a persisted terminal state as proof of recovery.
379    PersistenceFailed {
380        interrupted_turn_ids: Vec<TurnId>,
381        remaining_turn_ids: Vec<TurnId>,
382    },
383}
384
385#[derive(Clone)]
386struct QueuedTurnSteer {
387    message: UserMessage,
388    mailbox_ack: Option<MailboxDeliveryAck>,
389}
390
391#[derive(Clone)]
392struct MailboxDeliveryAck {
393    team_id: TeamId,
394    message_ids: Vec<String>,
395}
396
397#[derive(Clone)]
398struct InheritedTurnContext {
399    workspace: String,
400    instructions: InstructionBundle,
401    developer_context: Option<String>,
402}
403
404#[derive(Debug, Clone, Default, PartialEq, Eq)]
405pub struct ThreadActivity {
406    pub active_turn_id: Option<TurnId>,
407    pub active_flags: Vec<String>,
408}
409
410/// Per-thread settings persisted at thread creation and applied to every turn.
411#[derive(Debug, Clone, Default)]
412pub(crate) struct ThreadTurnOverrides {
413    pub(crate) tool_allowlist: Vec<String>,
414    pub(crate) developer_instructions: Option<String>,
415    pub(crate) external_tools: Vec<roder_api::tools::ToolSpec>,
416}
417
418#[derive(Debug, Clone, Copy, PartialEq, Eq)]
419pub(crate) enum TurnRunOutcome {
420    Completed,
421    Stopped,
422}
423
424impl PendingPlanExit {
425    pub fn new(
426        thread_id: ThreadId,
427        turn_id: TurnId,
428        request_id: String,
429        target_mode: PolicyMode,
430        plan_summary: Option<String>,
431        next_steps: Vec<String>,
432    ) -> Self {
433        let requested_at = OffsetDateTime::now_utc();
434        Self {
435            thread_id,
436            turn_id,
437            request_id,
438            target_mode,
439            plan_summary,
440            next_steps,
441            requested_at,
442            expires_at: Some(requested_at + default_plan_exit_timeout()),
443        }
444    }
445
446    pub fn is_expired(&self, now: OffsetDateTime) -> bool {
447        self.expires_at.is_some_and(|expires_at| now >= expires_at)
448    }
449}
450
451pub fn default_plan_exit_timeout() -> Duration {
452    Duration::minutes(10)
453}
454
455pub const DEFAULT_EXTERNAL_TOOL_TIMEOUT_SECONDS: u64 = 300;
456
457fn format_mailbox_messages(team: &TeamState, messages: &[TeamMailboxMessage]) -> String {
458    messages
459        .iter()
460        .map(|message| {
461            let recipient = team
462                .members
463                .iter()
464                .find(|member| member.id == message.to_member_id)
465                .map(canonical_team_member_path)
466                .unwrap_or_else(|| format!("/root/{}", message.to_member_id));
467            let sender = message
468                .from_member_id
469                .as_deref()
470                .and_then(|id| team.members.iter().find(|member| member.id == id))
471                .map(canonical_team_member_path)
472                .unwrap_or_else(|| "/root".to_string());
473            let message_type = match message.kind {
474                TeamMailboxMessageKind::Message => "MESSAGE",
475                TeamMailboxMessageKind::NewTask => "NEW_TASK",
476                TeamMailboxMessageKind::FinalAnswer => "FINAL_ANSWER",
477            };
478            format!(
479                "Message Type: {message_type}\nTask name: {recipient}\nSender: {sender}\nPayload:\n{}",
480                message.text
481            )
482        })
483        .collect::<Vec<_>>()
484        .join("\n\n")
485}
486
487fn canonical_team_member_path(member: &TeamMemberDescriptor) -> String {
488    if let Some(agent_path) = member
489        .agent_path
490        .as_deref()
491        .filter(|path| *path == "/root" || path.starts_with("/root/"))
492    {
493        return agent_path.to_string();
494    }
495    if member.role == TeamMemberRole::Lead {
496        return "/root".to_string();
497    }
498    member
499        .task_name
500        .as_deref()
501        .filter(|name| !name.trim().is_empty())
502        .map(|name| format!("/root/{}", name.trim().trim_matches('/')))
503        .unwrap_or_else(|| format!("/root/{}", member.id))
504}
505
506fn is_codex_v2_team(team: &TeamState) -> bool {
507    team.members.iter().any(|member| {
508        member
509            .model
510            .as_deref()
511            .is_some_and(|model| model_supports_reasoning_effort(model, REASONING_ULTRA))
512    })
513}
514
515fn runtime_local_team_view(
516    mut team: TeamState,
517    active_thread_ids: &std::collections::HashSet<ThreadId>,
518) -> TeamState {
519    for member in &mut team.members {
520        if member.status == TeamMemberStatus::Running
521            && !active_thread_ids.contains(&member.thread_id)
522        {
523            member.status = TeamMemberStatus::Interrupted;
524            member.current_turn_id = None;
525        }
526    }
527    team
528}
529
530type RunnerToolExecutionKey = (String, String);
531type RunnerToolExecutionMutex = Mutex<()>;
532type RunnerToolExecutionLockRegistry =
533    Mutex<HashMap<RunnerToolExecutionKey, Weak<RunnerToolExecutionMutex>>>;
534
535pub struct Runtime {
536    pub bus: EventBus,
537    pub registry: ExtensionRegistry,
538    config: RwLock<RuntimeConfig>,
539    pub(crate) execution_lease: Option<Arc<crate::execution_lease::RuntimeExecutionLease>>,
540    pending_plan_exit: RwLock<Option<PendingPlanExit>>,
541    pub(crate) pending_tool_approvals: Mutex<HashMap<String, PendingToolApproval>>,
542    pub(crate) pending_user_inputs: Mutex<HashMap<String, PendingUserInput>>,
543    pub(crate) pending_external_tool_calls: Mutex<HashMap<String, PendingExternalToolCall>>,
544    /// Serializes turn admission with shutdown quiescing. A drain takes this
545    /// gate before flipping `accepting_turns` and snapshotting active work so a
546    /// turn that observed the old flag cannot be inserted after the snapshot.
547    turn_admission: Mutex<()>,
548    active_turns: RwLock<HashMap<TurnId, ActiveTurnHandle>>,
549    turn_drains: RwLock<HashMap<TurnId, Arc<TurnDrainHandle>>>,
550    /// Provider cleanup handles register synchronously through the inference
551    /// tool-executor context. They are intentionally separate from active turn
552    /// admission so an interrupted turn can remain drainable after a follow-up
553    /// turn starts on the same thread.
554    provider_turn_cleanups: std::sync::Mutex<HashMap<TurnId, Arc<dyn ProviderTurnCleanup>>>,
555    accepting_turns: AtomicBool,
556    /// Monotonic counter used by bounded drains to surface lifecycle-state
557    /// persistence failures without making an individual provider turn fail.
558    lifecycle_persistence_failures: AtomicUsize,
559    lifecycle_shutdown_drains: AtomicUsize,
560    lifecycle_clean_shutdowns: AtomicUsize,
561    lifecycle_deadline_exceeded: AtomicUsize,
562    lifecycle_persistence_failed_drains: AtomicUsize,
563    lifecycle_restart_reconciliations: AtomicUsize,
564    lifecycle_shutdown_drain_duration_ms_total: AtomicUsize,
565    provider_cleanup_confirmed: AtomicUsize,
566    provider_cleanup_timed_out: AtomicUsize,
567    provider_cleanup_unknown: AtomicUsize,
568    active_turns_changed: Notify,
569    active_turn_selections: RwLock<HashMap<TurnId, ModelSelectionMode>>,
570    compaction_templates: RwLock<HashMap<ThreadId, AgentInferenceRequest>>,
571    thread_admission_gates: Mutex<HashMap<ThreadId, Arc<Mutex<()>>>>,
572    /// Threads that have already dispatched a `SessionStart` local hook.
573    ///
574    /// `ThreadCreated` fires once, but `ThreadLoaded` fires on every read of the
575    /// thread from its store, so without this a session ran its setup hooks
576    /// several times — including after the turn had already stopped.
577    session_hook_started: RwLock<HashSet<ThreadId>>,
578    active_turn_contexts: RwLock<HashMap<TurnId, InheritedTurnContext>>,
579    /// Process-local execution boundary. Local/CLI runtimes default to true;
580    /// hosted runtime pools set this false so missing runner metadata fails
581    /// closed instead of exposing the host workspace.
582    allow_local_workspaces: AtomicBool,
583    // Spawn-time live authority retained for every reusable turn of a long-lived teammate.
584    // This stays process-local because developer_context is explicitly volatile and must never
585    // be persisted to thread state.
586    team_member_turn_contexts: Mutex<HashMap<ThreadId, InheritedTurnContext>>,
587    workspace: PathBuf,
588    pub(crate) teams: TeamManager,
589    agent_team_spawn_lock: Mutex<()>,
590    pub(crate) roadmaps: Mutex<roder_roadmap::RoadmapRuntime>,
591    /// Completed reviews, most recent last, so a later publish can resolve a
592    /// review id back to its findings. Bounded; see `review::run`.
593    pub(crate) reviews: Mutex<Vec<crate::review::ReviewRecord>>,
594    pub(crate) goals: Arc<RuntimeGoalController>,
595    context_artifacts: roder_api::artifacts::ContextArtifactStore,
596    pub(crate) thread_store: Option<Arc<dyn ThreadStore>>,
597    thread_item_cache: Mutex<ThreadItemCache>,
598    pub(crate) tool_registry: ToolRegistry,
599    media_generation: Arc<crate::media_generation::MediaGenerationService>,
600    pub(crate) skills: RwLock<SkillRegistry>,
601    /// Lazily-started bounded dispatch of emitted events to registry
602    /// `EventSink`s (process extensions etc.); see `event_sink_dispatch`.
603    event_sink_dispatcher: tokio::sync::OnceCell<crate::event_sink_dispatch::EventSinkDispatcher>,
604    pub(crate) compaction_hysteresis:
605        std::sync::Mutex<HashMap<ThreadId, crate::compaction_runtime::CompactionState>>,
606    /// Per-thread agent-swarm mode overrides (roadmap 104). A thread present in
607    /// this map uses its stored value; absent threads fall back to the
608    /// runtime-global `RuntimeConfig.agent_swarm_mode` default. This mirrors the
609    /// team per-member policy-mode override idiom so swarm mode is per-thread
610    /// like the other `thread/*` operations, without a separate runtime.
611    agent_swarm_modes: RwLock<HashMap<ThreadId, bool>>,
612    /// Per-thread ultra mode overrides. A thread present in this map uses its
613    /// stored value; absent threads fall back to the runtime-global
614    /// `RuntimeConfig.ultra_mode` default.
615    ultra_modes: RwLock<HashMap<ThreadId, bool>>,
616    /**
617     * Live remote-runner sessions keyed by thread. Each per-thread mutex is
618     * held across provider initialization, making the first workspace-tool
619     * access a singleflight while allowing unrelated threads to initialize in
620     * parallel. Failed initialization is shared with current waiters, then
621     * evicted so a later tool call can retry.
622     */
623    runner_sessions: Arc<Mutex<HashMap<ThreadId, Arc<RunnerSessionSlot>>>>,
624    /**
625     * Outer tool-call locks keyed by the actual remote session, not the
626     * per-thread destination id. Hosted runtimes are tenant-scoped, so this
627     * serializes threads that share one sandbox without coupling different
628     * tenants or independent runner sessions.
629     */
630    runner_tool_execution_locks: RunnerToolExecutionLockRegistry,
631}
632
633#[derive(Clone)]
634struct CachedRunnerSession {
635    destination: RunnerDestination,
636    session: Arc<dyn RemoteRunnerSession>,
637}
638
639struct RunnerSessionSlot {
640    provider_id: String,
641    destination_id: String,
642    state: Mutex<RunnerSessionSlotState>,
643    initialized: Notify,
644}
645
646enum RunnerSessionSlotState {
647    Empty,
648    Initializing,
649    Ready(CachedRunnerSession),
650    Failed(Arc<str>),
651}
652
653impl RunnerSessionSlot {
654    fn empty(destination: &RunnerDestination) -> Self {
655        Self {
656            provider_id: destination.provider_id.clone(),
657            destination_id: destination.id.clone(),
658            state: Mutex::new(RunnerSessionSlotState::Empty),
659            initialized: Notify::new(),
660        }
661    }
662
663    fn ready(session: CachedRunnerSession) -> Self {
664        Self {
665            provider_id: session.destination.provider_id.clone(),
666            destination_id: session.destination.id.clone(),
667            state: Mutex::new(RunnerSessionSlotState::Ready(session)),
668            initialized: Notify::new(),
669        }
670    }
671
672    fn matches(&self, destination: &RunnerDestination) -> bool {
673        self.provider_id == destination.provider_id && self.destination_id == destination.id
674    }
675}
676
677impl Runtime {
678    pub fn new(registry: ExtensionRegistry, config: RuntimeConfig) -> anyhow::Result<Self> {
679        if registry.inference_engines.is_empty() {
680            anyhow::bail!("at least one inference engine must be registered");
681        }
682        validate_runtime_config_reasoning(&config)?;
683        validate_runtime_inference_router_config(&registry, &config)?;
684
685        let bus = EventBus::new(EVENT_BUS_CAPACITY);
686        let thread_store = registry
687            .thread_stores
688            .first()
689            .map(|factory| factory.create());
690        let mut tool_registry = ToolRegistry::default();
691        for contributor in &registry.tools {
692            contributor
693                .contribute(&mut tool_registry)
694                .with_context(|| format!("tool contributor {} failed", contributor.id()))?;
695        }
696        crate::agent_control_tools::contribute_agent_control_tools(&mut tool_registry)?;
697
698        let media_generation = Arc::new(crate::media_generation::MediaGenerationService::new(
699            registry.media_generator_providers.clone(),
700            config.media_generation.clone(),
701        ));
702        tool_registry.replace(Arc::new(
703            crate::media_generation::MediaGenerateImageTool::new(media_generation.clone()),
704        ));
705
706        let team_data_dir = config.team_data_dir.clone();
707        let workspace = config
708            .workspace
709            .clone()
710            .map(PathBuf::from)
711            .unwrap_or(std::env::current_dir()?);
712        let roadmap_data_dir = config
713            .roadmap_data_dir
714            .clone()
715            .unwrap_or_else(|| workspace.join(".roder"));
716        let context_artifacts = thread_store
717            .as_ref()
718            .and_then(|store| store.context_artifact_store())
719            .or_else(|| {
720                thread_store
721                    .as_ref()
722                    .and_then(|store| store.local_thread_root())
723                    .map(FilesystemContextArtifactStore::shared_thread_scoped)
724            })
725            .unwrap_or_else(|| {
726                FilesystemContextArtifactStore::shared_legacy(default_context_artifact_dir())
727            });
728        let goals = Arc::new(RuntimeGoalController::new(
729            bus.clone(),
730            thread_store.clone(),
731        ));
732        let runtime = Self {
733            bus,
734            registry,
735            config: RwLock::new(config),
736            execution_lease: None,
737            pending_plan_exit: RwLock::new(None),
738            pending_tool_approvals: Mutex::new(HashMap::new()),
739            pending_user_inputs: Mutex::new(HashMap::new()),
740            pending_external_tool_calls: Mutex::new(HashMap::new()),
741            turn_admission: Mutex::new(()),
742            active_turns: RwLock::new(HashMap::new()),
743            turn_drains: RwLock::new(HashMap::new()),
744            provider_turn_cleanups: std::sync::Mutex::new(HashMap::new()),
745            accepting_turns: AtomicBool::new(true),
746            lifecycle_persistence_failures: AtomicUsize::new(0),
747            lifecycle_shutdown_drains: AtomicUsize::new(0),
748            lifecycle_clean_shutdowns: AtomicUsize::new(0),
749            lifecycle_deadline_exceeded: AtomicUsize::new(0),
750            lifecycle_persistence_failed_drains: AtomicUsize::new(0),
751            lifecycle_restart_reconciliations: AtomicUsize::new(0),
752            lifecycle_shutdown_drain_duration_ms_total: AtomicUsize::new(0),
753            provider_cleanup_confirmed: AtomicUsize::new(0),
754            provider_cleanup_timed_out: AtomicUsize::new(0),
755            provider_cleanup_unknown: AtomicUsize::new(0),
756            active_turns_changed: Notify::new(),
757            active_turn_selections: RwLock::new(HashMap::new()),
758            compaction_templates: RwLock::new(HashMap::new()),
759            thread_admission_gates: Mutex::new(HashMap::new()),
760            session_hook_started: RwLock::new(HashSet::new()),
761            active_turn_contexts: RwLock::new(HashMap::new()),
762            allow_local_workspaces: AtomicBool::new(true),
763            team_member_turn_contexts: Mutex::new(HashMap::new()),
764            workspace: workspace.clone(),
765            teams: TeamManager::new(
766                team_data_dir.unwrap_or_else(crate::teams::default_team_data_dir),
767            ),
768            agent_team_spawn_lock: Mutex::new(()),
769            roadmaps: Mutex::new(roder_roadmap::RoadmapRuntime::new(
770                workspace,
771                roadmap_data_dir,
772            )),
773            reviews: Mutex::new(Vec::new()),
774            goals,
775            context_artifacts,
776            thread_store,
777            thread_item_cache: Mutex::new(ThreadItemCache::default()),
778            tool_registry,
779            media_generation,
780            skills: RwLock::new(SkillRegistry::load(SkillRegistryOptions::new(
781                PathBuf::new(),
782            ))),
783            event_sink_dispatcher: tokio::sync::OnceCell::new(),
784            compaction_hysteresis: crate::compaction_runtime::compaction_hysteresis_state(),
785            agent_swarm_modes: RwLock::new(HashMap::new()),
786            ultra_modes: RwLock::new(HashMap::new()),
787            runner_sessions: Arc::new(Mutex::new(HashMap::new())),
788            runner_tool_execution_locks: Mutex::new(HashMap::new()),
789        };
790        runtime.bus.emit(RoderEvent::RuntimeStarted(RuntimeStarted {
791            timestamp: OffsetDateTime::now_utc(),
792        }));
793        for manifest in &runtime.registry.manifests {
794            runtime
795                .bus
796                .emit(RoderEvent::ExtensionRegistered(ExtensionRegistered {
797                    extension_id: manifest.id.clone(),
798                    timestamp: OffsetDateTime::now_utc(),
799                }));
800        }
801        Ok(runtime)
802    }
803
804    pub fn from_engine(engine: Arc<dyn InferenceEngine>) -> anyhow::Result<Self> {
805        let mut builder = roder_api::extension::ExtensionRegistryBuilder::new();
806        builder.inference_engine(engine);
807        Self::new(builder.build()?, RuntimeConfig::default())
808    }
809
810    pub fn fake() -> anyhow::Result<Self> {
811        Self::from_engine(Arc::new(FakeInferenceEngine))
812    }
813
814    pub fn subscribe_events(&self) -> tokio::sync::broadcast::Receiver<EventEnvelope> {
815        self.bus.subscribe()
816    }
817
818    /// Returns process-local, redacted lifecycle health counters. The snapshot
819    /// intentionally has fixed fields and contains no provider, command,
820    /// process, thread, or turn identifiers.
821    pub fn lifecycle_metrics(&self) -> LifecycleMetricsSnapshot {
822        LifecycleMetricsSnapshot {
823            shutdown_drain_count: self.lifecycle_shutdown_drains.load(Ordering::Acquire) as u64,
824            clean_shutdown_count: self.lifecycle_clean_shutdowns.load(Ordering::Acquire) as u64,
825            deadline_exceeded_count: self.lifecycle_deadline_exceeded.load(Ordering::Acquire)
826                as u64,
827            persistence_failed_count: self
828                .lifecycle_persistence_failed_drains
829                .load(Ordering::Acquire) as u64,
830            restart_reconciliation_count: self
831                .lifecycle_restart_reconciliations
832                .load(Ordering::Acquire) as u64,
833            lifecycle_persistence_failure_count: self
834                .lifecycle_persistence_failures
835                .load(Ordering::Acquire) as u64,
836            shutdown_drain_duration_ms_total: self
837                .lifecycle_shutdown_drain_duration_ms_total
838                .load(Ordering::Acquire) as u64,
839            provider_cleanup_confirmed_count: self
840                .provider_cleanup_confirmed
841                .load(Ordering::Acquire) as u64,
842            provider_cleanup_timed_out_count: self
843                .provider_cleanup_timed_out
844                .load(Ordering::Acquire) as u64,
845            provider_cleanup_unknown_count: self.provider_cleanup_unknown.load(Ordering::Acquire)
846                as u64,
847        }
848    }
849
850    async fn persist_turn_lifecycle_record(&self, record: &TurnLifecycleRecord) -> bool {
851        let Some(store) = &self.thread_store else {
852            return true;
853        };
854        let state = match record.extension_state() {
855            Ok(state) => state,
856            Err(_) => {
857                self.lifecycle_persistence_failures
858                    .fetch_add(1, Ordering::AcqRel);
859                return false;
860            }
861        };
862        // Lifecycle diagnostics must not turn a provider result into a failed
863        // turn merely because a third-party store lacks extension-state support.
864        // JSONL/Postgres stores persist this append-only record atomically at
865        // their own storage boundary.
866        if store
867            .append_extension_state(&record.thread_id, &state)
868            .await
869            .is_err()
870        {
871            self.lifecycle_persistence_failures
872                .fetch_add(1, Ordering::AcqRel);
873            return false;
874        }
875        true
876    }
877
878    async fn record_turn_lifecycle(
879        &self,
880        thread_id: ThreadId,
881        turn_id: TurnId,
882        state: TurnLifecycleState,
883        cleanup: TurnCleanupState,
884        reason: Option<TurnLifecycleReason>,
885    ) -> TurnLifecycleRecord {
886        self.record_turn_lifecycle_with_ownership(
887            thread_id,
888            turn_id,
889            state,
890            cleanup,
891            reason,
892            TurnCleanupOwnership::RuntimeTaskOnly,
893        )
894        .await
895    }
896
897    async fn record_turn_lifecycle_with_ownership(
898        &self,
899        thread_id: ThreadId,
900        turn_id: TurnId,
901        state: TurnLifecycleState,
902        cleanup: TurnCleanupState,
903        reason: Option<TurnLifecycleReason>,
904        ownership: TurnCleanupOwnership,
905    ) -> TurnLifecycleRecord {
906        let record = TurnLifecycleRecord::new(
907            thread_id,
908            turn_id,
909            state,
910            cleanup,
911            reason,
912            OffsetDateTime::now_utc(),
913        )
914        .with_ownership(ownership);
915        let _ = self.persist_turn_lifecycle_record(&record).await;
916        self.emit(RoderEvent::TurnLifecycleUpdated(record.clone()))
917            .await;
918        record
919    }
920
921    pub(crate) fn register_provider_turn_cleanup(
922        &self,
923        turn_id: &TurnId,
924        cleanup: Arc<dyn ProviderTurnCleanup>,
925    ) {
926        if let Ok(mut cleanups) = self.provider_turn_cleanups.lock() {
927            cleanups.insert(turn_id.clone(), cleanup);
928        }
929    }
930
931    fn provider_cleanup_ownership(&self, turn_id: &TurnId) -> TurnCleanupOwnership {
932        self.provider_turn_cleanups
933            .lock()
934            .ok()
935            .and_then(|cleanups| cleanups.get(turn_id).cloned())
936            .map(|cleanup| cleanup.ownership())
937            .unwrap_or(TurnCleanupOwnership::RuntimeTaskOnly)
938    }
939
940    async fn await_provider_turn_cleanup(
941        &self,
942        turn_id: &TurnId,
943    ) -> (TurnCleanupState, TurnCleanupOwnership) {
944        let cleanup = self
945            .provider_turn_cleanups
946            .lock()
947            .ok()
948            .and_then(|mut cleanups| cleanups.remove(turn_id));
949        let Some(cleanup) = cleanup else {
950            return (
951                TurnCleanupState::Unknown,
952                TurnCleanupOwnership::RuntimeTaskOnly,
953            );
954        };
955        let ownership = cleanup.ownership();
956        match tokio::time::timeout(PROVIDER_CLEANUP_TIMEOUT, cleanup.wait_for_cleanup()).await {
957            Ok(Ok(())) => {
958                self.provider_cleanup_confirmed
959                    .fetch_add(1, Ordering::AcqRel);
960                (
961                    TurnCleanupState::Completed,
962                    TurnCleanupOwnership::ProviderCleanupConfirmed,
963                )
964            }
965            Ok(Err(_)) => {
966                self.provider_cleanup_unknown.fetch_add(1, Ordering::AcqRel);
967                (TurnCleanupState::Unknown, ownership)
968            }
969            Err(_) => {
970                self.provider_cleanup_timed_out
971                    .fetch_add(1, Ordering::AcqRel);
972                (TurnCleanupState::TimedOut, ownership)
973            }
974        }
975    }
976
977    fn record_lifecycle_drain_outcome(
978        &self,
979        started_at: tokio::time::Instant,
980        outcome: &RuntimeDrainOutcome,
981    ) {
982        self.lifecycle_shutdown_drains
983            .fetch_add(1, Ordering::AcqRel);
984        self.lifecycle_shutdown_drain_duration_ms_total.fetch_add(
985            started_at.elapsed().as_millis().min(usize::MAX as u128) as usize,
986            Ordering::AcqRel,
987        );
988        match outcome {
989            RuntimeDrainOutcome::Clean { .. } => {
990                self.lifecycle_clean_shutdowns
991                    .fetch_add(1, Ordering::AcqRel);
992            }
993            RuntimeDrainOutcome::DeadlineExceeded { .. } => {
994                self.lifecycle_deadline_exceeded
995                    .fetch_add(1, Ordering::AcqRel);
996            }
997            RuntimeDrainOutcome::PersistenceFailed { .. } => {
998                self.lifecycle_persistence_failed_drains
999                    .fetch_add(1, Ordering::AcqRel);
1000            }
1001        }
1002    }
1003
1004    fn lifecycle_record_for_event(event: &RoderEvent) -> Option<TurnLifecycleRecord> {
1005        match event {
1006            RoderEvent::TurnStarted(event) => Some(TurnLifecycleRecord::new(
1007                event.thread_id.clone(),
1008                event.turn_id.clone(),
1009                TurnLifecycleState::Running,
1010                TurnCleanupState::NotRequested,
1011                None,
1012                event.timestamp,
1013            )),
1014            // `run_turn` persists completed turns after it has awaited a
1015            // registered provider cleanup handle. Do not let the generic event
1016            // projection overwrite that acknowledgement with an unproven state.
1017            RoderEvent::TurnCompleted(_) => None,
1018            // Some failure paths already have an event before the turn wrapper
1019            // reaches its cleanup acknowledgement. Preserve a durable fallback
1020            // reason here; the wrapper appends a later record with the more
1021            // precise cleanup outcome when a provider registered one.
1022            RoderEvent::TurnFailed(event) => Some(TurnLifecycleRecord::new(
1023                event.thread_id.clone(),
1024                event.turn_id.clone(),
1025                TurnLifecycleState::Failed,
1026                TurnCleanupState::Unknown,
1027                Some(if event.error_kind.as_deref() == Some("deadline_timeout") {
1028                    TurnLifecycleReason::DeadlineExceeded
1029                } else {
1030                    TurnLifecycleReason::ProviderFailure
1031                }),
1032                event.timestamp,
1033            )),
1034            RoderEvent::TurnDeadlineExceeded(_) | RoderEvent::TurnInterrupted(_) => None,
1035            RoderEvent::TurnLifecycleUpdated(_) => None,
1036            _ => None,
1037        }
1038    }
1039
1040    pub fn registry(&self) -> &ExtensionRegistry {
1041        &self.registry
1042    }
1043
1044    pub fn media_generation(&self) -> Arc<crate::media_generation::MediaGenerationService> {
1045        self.media_generation.clone()
1046    }
1047
1048    pub fn context_artifacts(&self) -> roder_api::artifacts::ContextArtifactStore {
1049        self.context_artifacts.clone()
1050    }
1051
1052    pub async fn execute_workflow_tool(
1053        &self,
1054        thread_id: ThreadId,
1055        tool_name: &str,
1056        arguments: serde_json::Value,
1057    ) -> anyhow::Result<ToolResult> {
1058        let Some(executor) = self.tool_registry.get(tool_name) else {
1059            anyhow::bail!("tool not found: {tool_name}");
1060        };
1061        let tool_call = ToolCall {
1062            id: format!("slash-{tool_name}"),
1063            name: tool_name.to_string(),
1064            raw_arguments: serde_json::to_string(&arguments)?,
1065            arguments,
1066            thread_id: thread_id.clone(),
1067            turn_id: "slash-command".to_string(),
1068        };
1069        let runtime_config = self.status().await;
1070        let ctx = self.tool_execution_context(
1071            thread_id,
1072            "slash-command".to_string(),
1073            runtime_config.policy_mode,
1074            runtime_config.workspace.as_deref(),
1075            Some(&runtime_config.command_shell),
1076        );
1077        executor.execute(ctx, tool_call).await
1078    }
1079
1080    pub(crate) fn tool_execution_context(
1081        &self,
1082        thread_id: ThreadId,
1083        turn_id: TurnId,
1084        mode: PolicyMode,
1085        workspace: Option<&str>,
1086        command_shell: Option<&str>,
1087    ) -> ToolExecutionContext {
1088        let mut ctx = ToolExecutionContext::new(thread_id, turn_id, mode)
1089            .with_command_shell(command_shell.unwrap_or_default())
1090            .with_process_runner(Arc::new(LocalProcessRunner))
1091            .with_context_artifacts(self.context_artifacts.backend())
1092            .with_goal_controller(self.goals.clone())
1093            .with_subagent_trace_sink(Arc::new(RuntimeSubagentTraceSink::new(
1094                self.bus.clone(),
1095                self.thread_store.clone(),
1096            )))
1097            .with_swarm_progress_sink(Arc::new(RuntimeAgentSwarmProgressSink::new(
1098                self.bus.clone(),
1099                self.thread_store.clone(),
1100            )));
1101        if let Some(workspace) = workspace {
1102            ctx = ctx.with_workspace_handle(Arc::new(ScopedFilesystem::new(workspace)));
1103        }
1104        ctx
1105    }
1106
1107    pub async fn status(&self) -> RuntimeConfig {
1108        self.config.read().await.clone()
1109    }
1110
1111    pub async fn set_skills(&self, skills: SkillRegistry) {
1112        *self.skills.write().await = skills;
1113    }
1114
1115    pub async fn skills_snapshot(&self) -> SkillRegistry {
1116        self.skills.read().await.clone()
1117    }
1118
1119    pub fn workspace(&self) -> PathBuf {
1120        self.workspace.clone()
1121    }
1122
1123    /// Controls whether native workspace tools may execute without an
1124    /// explicit remote-runner binding. Hosted runtimes disable this after
1125    /// construction; ordinary local runtimes retain the compatible default.
1126    pub fn set_allow_local_workspaces(&self, allowed: bool) {
1127        self.allow_local_workspaces
1128            .store(allowed, Ordering::Relaxed);
1129    }
1130
1131    pub fn allows_local_workspaces(&self) -> bool {
1132        self.allow_local_workspaces.load(Ordering::Relaxed)
1133    }
1134
1135    pub async fn set_remote_runner_destination(&self, destination: Option<RunnerDestination>) {
1136        let lifecycle = destination.as_ref().map(|destination| RunnerLifecycle {
1137            destination_id: destination.id.clone(),
1138            provider_id: destination.provider_id.clone(),
1139            state: "configured".to_string(),
1140            session_id: None,
1141            timestamp: OffsetDateTime::now_utc(),
1142        });
1143        self.config.write().await.remote_runner_destination = destination;
1144        if let Some(lifecycle) = lifecycle {
1145            self.emit(RoderEvent::RunnerLifecycle(lifecycle)).await;
1146        } else {
1147            self.emit(RoderEvent::RunnerLifecycle(RunnerLifecycle {
1148                destination_id: "local".to_string(),
1149                provider_id: "local".to_string(),
1150                state: "local_fallback".to_string(),
1151                session_id: None,
1152                timestamp: OffsetDateTime::now_utc(),
1153            }))
1154            .await;
1155        }
1156    }
1157
1158    pub async fn set_file_backed_dynamic_context(&self, enabled: bool) -> RuntimeConfig {
1159        let mut cfg = self.config.write().await;
1160        cfg.file_backed_dynamic_context = enabled;
1161        cfg.clone()
1162    }
1163
1164    pub async fn set_command_shell(&self, shell: String) -> RuntimeConfig {
1165        let mut cfg = self.config.write().await;
1166        cfg.command_shell = shell;
1167        cfg.clone()
1168    }
1169
1170    pub async fn pending_plan_exit(&self) -> Option<PendingPlanExit> {
1171        let mut pending = self.pending_plan_exit.write().await;
1172        let current = pending.clone()?;
1173        if !current.is_expired(OffsetDateTime::now_utc()) {
1174            return Some(current);
1175        }
1176        *pending = None;
1177        drop(pending);
1178        self.emit_plan_exit_resolved(&current, false, self.status().await.policy_mode)
1179            .await;
1180        None
1181    }
1182
1183    pub async fn set_policy_mode(
1184        &self,
1185        mode: PolicyMode,
1186        reason: Option<String>,
1187    ) -> anyhow::Result<RuntimeConfig> {
1188        let mut cfg = self.config.write().await;
1189        let previous_mode = cfg.policy_mode;
1190        cfg.policy_mode = mode;
1191        let next = cfg.clone();
1192        drop(cfg);
1193        self.emit(RoderEvent::PolicyModeChanged(PolicyModeChanged {
1194            thread_id: "runtime".to_string(),
1195            turn_id: None,
1196            previous_mode,
1197            new_mode: mode,
1198            reason,
1199            timestamp: OffsetDateTime::now_utc(),
1200        }))
1201        .await;
1202        self.auto_resolve_pending_tool_approvals().await;
1203        Ok(next)
1204    }
1205
1206    /// Toggle agent-swarm mode for the runtime (roadmap 104). When enabled, the
1207    /// swarm reminder is injected into each turn's developer instructions so
1208    /// every app-server/SDK client gets it, not only the TUI.
1209    pub async fn set_agent_swarm_mode(
1210        &self,
1211        enabled: bool,
1212        trigger: roder_api::subagents::AgentSwarmModeTrigger,
1213    ) -> anyhow::Result<RuntimeConfig> {
1214        let mut cfg = self.config.write().await;
1215        cfg.agent_swarm_mode = enabled;
1216        let next = cfg.clone();
1217        drop(cfg);
1218        self.emit(RoderEvent::AgentSwarmModeChanged(
1219            roder_api::subagents::AgentSwarmModeChanged {
1220                thread_id: "runtime".to_string(),
1221                turn_id: None,
1222                enabled,
1223                trigger,
1224                timestamp: OffsetDateTime::now_utc(),
1225            },
1226        ))
1227        .await;
1228        Ok(next)
1229    }
1230
1231    /// Toggle agent-swarm mode for a single thread (roadmap 104). The override is
1232    /// stored per thread so toggling swarm mode on one thread does not leak the
1233    /// reminder into other threads sharing the runtime; absent threads fall back
1234    /// to the runtime-global default. The emitted `AgentSwarmModeChanged` event
1235    /// carries the real `thread_id`.
1236    pub async fn set_agent_swarm_mode_for_thread(
1237        &self,
1238        thread_id: &str,
1239        enabled: bool,
1240        trigger: roder_api::subagents::AgentSwarmModeTrigger,
1241    ) -> bool {
1242        {
1243            let mut modes = self.agent_swarm_modes.write().await;
1244            modes.insert(thread_id.to_string(), enabled);
1245        }
1246        self.emit(RoderEvent::AgentSwarmModeChanged(
1247            roder_api::subagents::AgentSwarmModeChanged {
1248                thread_id: thread_id.to_string(),
1249                turn_id: None,
1250                enabled,
1251                trigger,
1252                timestamp: OffsetDateTime::now_utc(),
1253            },
1254        ))
1255        .await;
1256        enabled
1257    }
1258
1259    /// Resolve whether agent-swarm mode is active for `thread_id`: the per-thread
1260    /// override when present, otherwise the runtime-global default. The turn loop
1261    /// uses this to decide whether to inject the swarm reminder.
1262    pub async fn effective_agent_swarm_mode_for_thread(&self, thread_id: &str) -> bool {
1263        if let Some(enabled) = self.agent_swarm_modes.read().await.get(thread_id).copied() {
1264            return enabled;
1265        }
1266        self.status().await.agent_swarm_mode
1267    }
1268
1269    /// Toggle ultra mode for the runtime. When enabled, proactive multi-agent
1270    /// policy is injected into each turn's developer instructions for any
1271    /// model, so every app-server/SDK client gets it (not only the TUI).
1272    pub async fn set_ultra_mode(
1273        &self,
1274        enabled: bool,
1275        trigger: roder_api::subagents::UltraModeTrigger,
1276    ) -> anyhow::Result<RuntimeConfig> {
1277        let mut cfg = self.config.write().await;
1278        cfg.ultra_mode = enabled;
1279        let next = cfg.clone();
1280        drop(cfg);
1281        self.emit(RoderEvent::UltraModeChanged(
1282            roder_api::subagents::UltraModeChanged {
1283                thread_id: "runtime".to_string(),
1284                turn_id: None,
1285                enabled,
1286                trigger,
1287                timestamp: OffsetDateTime::now_utc(),
1288            },
1289        ))
1290        .await;
1291        Ok(next)
1292    }
1293
1294    /// Toggle ultra mode for a single thread. The override is stored per thread
1295    /// so toggling on one thread does not leak proactive multi-agent policy into
1296    /// other threads sharing the runtime; absent threads fall back to the
1297    /// runtime-global default.
1298    pub async fn set_ultra_mode_for_thread(
1299        &self,
1300        thread_id: &str,
1301        enabled: bool,
1302        trigger: roder_api::subagents::UltraModeTrigger,
1303    ) -> bool {
1304        {
1305            let mut modes = self.ultra_modes.write().await;
1306            modes.insert(thread_id.to_string(), enabled);
1307        }
1308        self.emit(RoderEvent::UltraModeChanged(
1309            roder_api::subagents::UltraModeChanged {
1310                thread_id: thread_id.to_string(),
1311                turn_id: None,
1312                enabled,
1313                trigger,
1314                timestamp: OffsetDateTime::now_utc(),
1315            },
1316        ))
1317        .await;
1318        enabled
1319    }
1320
1321    /// Resolve whether ultra mode is active for `thread_id`: the per-thread
1322    /// override when present, otherwise the runtime-global default. The turn
1323    /// loop uses this (together with Sol/Terra Ultra effort) to decide whether
1324    /// to inject proactive multi-agent policy.
1325    pub async fn effective_ultra_mode_for_thread(&self, thread_id: &str) -> bool {
1326        if let Some(enabled) = self.ultra_modes.read().await.get(thread_id).copied() {
1327            return enabled;
1328        }
1329        self.status().await.ultra_mode
1330    }
1331
1332    pub async fn set_hosted_web_search(
1333        &self,
1334        mode: HostedWebSearchMode,
1335    ) -> anyhow::Result<RuntimeConfig> {
1336        let mut cfg = self.config.write().await;
1337        cfg.hosted_web_search = HostedWebSearchConfig { mode };
1338        Ok(cfg.clone())
1339    }
1340
1341    pub async fn record_pending_plan_exit(&self, pending: PendingPlanExit) {
1342        *self.pending_plan_exit.write().await = Some(pending.clone());
1343        self.emit(RoderEvent::PolicyExitPlanRequested(
1344            PolicyExitPlanRequested {
1345                thread_id: pending.thread_id,
1346                turn_id: pending.turn_id,
1347                request_id: pending.request_id,
1348                target_mode: pending.target_mode,
1349                plan_summary: pending.plan_summary,
1350                next_steps: pending.next_steps,
1351                timestamp: OffsetDateTime::now_utc(),
1352            },
1353        ))
1354        .await;
1355    }
1356
1357    pub async fn resolve_pending_plan_exit(
1358        &self,
1359        request_id: &str,
1360        approved: bool,
1361    ) -> anyhow::Result<Option<PendingPlanExit>> {
1362        let mut pending = self.pending_plan_exit.write().await;
1363        let Some(current) = pending.clone() else {
1364            return Ok(None);
1365        };
1366        if current.request_id != request_id {
1367            anyhow::bail!("pending plan exit request {request_id:?} was not found");
1368        }
1369        *pending = None;
1370        drop(pending);
1371
1372        let approved = approved && !current.is_expired(OffsetDateTime::now_utc());
1373        let resolved_mode = if approved {
1374            let mut cfg = self.config.write().await;
1375            let previous_mode = cfg.policy_mode;
1376            cfg.policy_mode = current.target_mode;
1377            drop(cfg);
1378            self.emit(RoderEvent::PolicyModeChanged(PolicyModeChanged {
1379                thread_id: current.thread_id.clone(),
1380                turn_id: Some(current.turn_id.clone()),
1381                previous_mode,
1382                new_mode: current.target_mode,
1383                reason: Some("approved plan exit".to_string()),
1384                timestamp: OffsetDateTime::now_utc(),
1385            }))
1386            .await;
1387            self.auto_resolve_pending_tool_approvals().await;
1388            current.target_mode
1389        } else {
1390            self.status().await.policy_mode
1391        };
1392        self.emit_plan_exit_resolved(&current, approved, resolved_mode)
1393            .await;
1394        Ok(Some(current))
1395    }
1396
1397    pub async fn resolve_tool_approval(
1398        &self,
1399        approval_id: &str,
1400        approved: bool,
1401    ) -> anyhow::Result<bool> {
1402        let pending = self.pending_tool_approvals.lock().await.remove(approval_id);
1403        let Some(pending) = pending else {
1404            return Ok(false);
1405        };
1406        self.emit(RoderEvent::ApprovalResolved(ApprovalResolved {
1407            thread_id: pending.thread_id,
1408            turn_id: pending.turn_id,
1409            approval_id: approval_id.to_string(),
1410            tool_id: pending.tool_id,
1411            tool_name: pending.tool_name,
1412            approved,
1413            timestamp: OffsetDateTime::now_utc(),
1414        }))
1415        .await;
1416        let _ = pending.tx.send(approved);
1417        Ok(true)
1418    }
1419
1420    /// Completes a pending host-executed tool call (`tools/resolve`). Returns false when the
1421    /// request id is unknown, already resolved, timed out, or cancelled by a turn interrupt.
1422    pub async fn resolve_external_tool_call(
1423        &self,
1424        request_id: &str,
1425        resolution: ExternalToolResolution,
1426    ) -> anyhow::Result<bool> {
1427        let mut calls = self.pending_external_tool_calls.lock().await;
1428        let Some(pending) = calls.get(request_id) else {
1429            return Ok(false);
1430        };
1431        self.emit_external_execution(RoderEvent::ExternalToolCallResolved(
1432            ExternalToolCallResolved {
1433                thread_id: pending.thread_id.clone(),
1434                turn_id: pending.turn_id.clone(),
1435                request_id: request_id.to_string(),
1436                tool_id: pending.tool_id.clone(),
1437                tool_name: pending.tool_name.clone(),
1438                outcome: ExternalToolCallOutcome::Resolved,
1439                is_error: resolution.is_error,
1440                timestamp: OffsetDateTime::now_utc(),
1441            },
1442        ))
1443        .await?;
1444        let Some(pending) = calls.remove(request_id) else {
1445            return Ok(false);
1446        };
1447        drop(calls);
1448        let _ = pending.tx.send(resolution);
1449        Ok(true)
1450    }
1451
1452    /// Drops every pending external tool call for the turn and reports them as cancelled so an
1453    /// interrupt does not leak requests waiting on `tools/resolve`.
1454    async fn cancel_pending_external_tool_calls_for_turn(&self, turn_id: &TurnId) {
1455        let cancelled = {
1456            let mut pending = self.pending_external_tool_calls.lock().await;
1457            let request_ids = pending
1458                .iter()
1459                .filter(|(_, call)| &call.turn_id == turn_id)
1460                .map(|(request_id, _)| request_id.clone())
1461                .collect::<Vec<_>>();
1462            request_ids
1463                .into_iter()
1464                .filter_map(|request_id| pending.remove(&request_id).map(|call| (request_id, call)))
1465                .collect::<Vec<_>>()
1466        };
1467        for (request_id, call) in cancelled {
1468            self.emit(RoderEvent::ExternalToolCallResolved(
1469                ExternalToolCallResolved {
1470                    thread_id: call.thread_id,
1471                    turn_id: call.turn_id,
1472                    request_id,
1473                    tool_id: call.tool_id,
1474                    tool_name: call.tool_name,
1475                    outcome: ExternalToolCallOutcome::Cancelled,
1476                    is_error: true,
1477                    timestamp: OffsetDateTime::now_utc(),
1478                },
1479            ))
1480            .await;
1481        }
1482    }
1483
1484    pub async fn resolve_user_input(
1485        &self,
1486        request_id: &str,
1487        answers: serde_json::Value,
1488    ) -> anyhow::Result<bool> {
1489        let pending = self.pending_user_inputs.lock().await.remove(request_id);
1490        let Some(pending) = pending else {
1491            return Ok(false);
1492        };
1493        self.emit(RoderEvent::UserInputResolved(UserInputResolved {
1494            thread_id: pending.thread_id,
1495            turn_id: pending.turn_id,
1496            request_id: request_id.to_string(),
1497            answers: answers.clone(),
1498            timestamp: OffsetDateTime::now_utc(),
1499        }))
1500        .await;
1501        let _ = pending.tx.send(answers);
1502        Ok(true)
1503    }
1504
1505    async fn emit_plan_exit_resolved(
1506        &self,
1507        current: &PendingPlanExit,
1508        approved: bool,
1509        resolved_mode: PolicyMode,
1510    ) {
1511        self.emit(RoderEvent::PolicyExitPlanResolved(PolicyExitPlanResolved {
1512            thread_id: current.thread_id.clone(),
1513            turn_id: current.turn_id.clone(),
1514            request_id: current.request_id.clone(),
1515            approved,
1516            target_mode: current.target_mode,
1517            resolved_mode,
1518            timestamp: OffsetDateTime::now_utc(),
1519        }))
1520        .await;
1521    }
1522
1523    pub async fn select_provider(
1524        &self,
1525        provider: String,
1526        model: Option<String>,
1527        reasoning: Option<String>,
1528    ) -> anyhow::Result<RuntimeConfig> {
1529        let next = self
1530            .preview_provider_selection(provider, model, reasoning)
1531            .await?;
1532        let mut cfg = self.config.write().await;
1533        *cfg = next;
1534        Ok(cfg.clone())
1535    }
1536
1537    pub async fn preview_provider_selection(
1538        &self,
1539        provider: String,
1540        model: Option<String>,
1541        reasoning: Option<String>,
1542    ) -> anyhow::Result<RuntimeConfig> {
1543        self.engine_for(&provider)?;
1544        let mut cfg = self.config.read().await.clone();
1545        cfg.default_provider = provider;
1546        if let Some(model) = model {
1547            cfg.default_model = model;
1548        }
1549        if let Some(reasoning) = reasoning {
1550            if reasoning == REASONING_NONE
1551                && !model_supports_reasoning(&cfg.default_model, &reasoning)
1552            {
1553                return Ok(cfg.clone());
1554            }
1555            validate_reasoning_effort(&cfg.default_model, &reasoning)?;
1556            cfg.reasoning = Some(reasoning);
1557        }
1558        Ok(cfg)
1559    }
1560
1561    pub async fn effective_reasoning(&self) -> String {
1562        let cfg = self.config.read().await;
1563        effective_reasoning_for_model(&cfg, &cfg.default_model)
1564    }
1565
1566    pub fn effective_reasoning_for_config(cfg: &RuntimeConfig) -> String {
1567        effective_reasoning_for_model(cfg, &cfg.default_model)
1568    }
1569
1570    pub async fn set_dynamic_workflow_effort(
1571        &self,
1572        effort_profile: DynamicWorkflowEffortProfile,
1573    ) -> RuntimeConfig {
1574        let mut cfg = self.config.write().await;
1575        cfg.dynamic_workflows.effort_profile = effort_profile;
1576        cfg.clone()
1577    }
1578
1579    pub async fn dynamic_workflow_trigger_decision(
1580        &self,
1581        message: &str,
1582    ) -> WorkflowTriggerDecision {
1583        let cfg = self.config.read().await;
1584        classify_workflow_trigger(message, &cfg.dynamic_workflows)
1585    }
1586
1587    pub async fn create_thread(&self, title: Option<String>) -> anyhow::Result<ThreadMetadata> {
1588        self.create_thread_with(CreateThreadRequest {
1589            title,
1590            workspace: self.workspace.display().to_string(),
1591            workspace_id: None,
1592            root_id: None,
1593            provider: None,
1594            model: None,
1595            selection_mode: None,
1596            tool_allowlist: Vec::new(),
1597            developer_instructions: None,
1598            external_tools: Vec::new(),
1599            runner: None,
1600        })
1601        .await
1602    }
1603
1604    /**
1605     * Resolves an explicit thread-runner selection into a persisted binding.
1606     * Fails fast at thread creation when the provider is missing or rejects
1607     * the destination, instead of surfacing the error on the first tool call.
1608     */
1609    async fn resolve_thread_runner_binding(
1610        &self,
1611        thread_id: &str,
1612        selection: ThreadRunnerSelection,
1613    ) -> anyhow::Result<ThreadRunnerBinding> {
1614        let provider = self
1615            .registry
1616            .remote_runner_providers
1617            .iter()
1618            .find(|provider| provider.id() == selection.provider_id)
1619            .cloned()
1620            .ok_or_else(|| {
1621                anyhow::anyhow!(
1622                    "remote runner provider {:?} is not installed",
1623                    selection.provider_id
1624                )
1625            })?;
1626        let workspace = selection.workspace.trim();
1627        anyhow::ensure!(
1628            std::path::Path::new(workspace).is_absolute(),
1629            "runner workspace must be an absolute path on the runner: {workspace:?}"
1630        );
1631        let mut read_roots = Vec::with_capacity(selection.read_roots.len());
1632        for read_root in &selection.read_roots {
1633            let trimmed = read_root.trim();
1634            anyhow::ensure!(
1635                std::path::Path::new(trimmed).is_absolute(),
1636                "runner read root must be an absolute path on the runner: {trimmed:?}"
1637            );
1638            read_roots.push(PathBuf::from(trimmed));
1639        }
1640        let destination = RunnerDestination {
1641            id: format!("thread-{thread_id}"),
1642            provider_id: selection.provider_id,
1643            config: selection.config,
1644            default_manifest: roder_api::remote_runner::RunnerManifest::default(),
1645        };
1646        provider.validate_destination(&destination).await?;
1647        Ok(ThreadRunnerBinding {
1648            destination,
1649            workspace: PathBuf::from(workspace),
1650            read_roots,
1651        })
1652    }
1653
1654    /**
1655     * Build a per-thread binding from a runtime-level destination (selected via
1656     * the TUI runner picker or config `default_destination`) when the
1657     * destination's provider advertises a default workspace. Returns `None` when
1658     * there is no destination or the provider opts out (no default workspace),
1659     * preserving the legacy behavior where such threads keep local tools.
1660     */
1661    async fn synthesize_runtime_runner_binding(
1662        &self,
1663        thread_id: &str,
1664        destination: Option<RunnerDestination>,
1665    ) -> anyhow::Result<Option<ThreadRunnerBinding>> {
1666        let Some(destination) = destination else {
1667            return Ok(None);
1668        };
1669        let Some(provider) = self
1670            .registry
1671            .remote_runner_providers
1672            .iter()
1673            .find(|provider| provider.id() == destination.provider_id)
1674        else {
1675            return Ok(None);
1676        };
1677        // An explicit workspace in the destination config wins over the
1678        // provider default; absence of both means the provider opts out.
1679        let workspace = destination
1680            .config
1681            .get("working_dir")
1682            .and_then(serde_json::Value::as_str)
1683            .map(str::to_string)
1684            .or_else(|| provider.default_workspace());
1685        let Some(workspace) = workspace else {
1686            return Ok(None);
1687        };
1688        let selection = ThreadRunnerSelection {
1689            provider_id: destination.provider_id.clone(),
1690            config: destination.config.clone(),
1691            workspace,
1692            read_roots: Vec::new(),
1693        };
1694        Ok(Some(
1695            self.resolve_thread_runner_binding(thread_id, selection)
1696                .await?,
1697        ))
1698    }
1699
1700    /// Validates a runner selection without creating a thread; the placeholder destination id only appears in error messages.
1701    pub async fn validate_thread_runner_selection(
1702        &self,
1703        selection: ThreadRunnerSelection,
1704    ) -> anyhow::Result<()> {
1705        self.resolve_thread_runner_binding("validate", selection)
1706            .await
1707            .map(|_| ())
1708    }
1709
1710    pub async fn create_thread_with(
1711        &self,
1712        req: CreateThreadRequest,
1713    ) -> anyhow::Result<ThreadMetadata> {
1714        tool_advertisement::validate_external_tool_names(&req.external_tools)?;
1715        let cfg = self.config.read().await.clone();
1716        let now = OffsetDateTime::now_utc();
1717        let workspace = validate_thread_workspace(&req.workspace)?;
1718        let provider = req.provider.unwrap_or(cfg.default_provider);
1719        let model = req.model.unwrap_or(cfg.default_model);
1720        let selection_mode = req
1721            .selection_mode
1722            .unwrap_or_else(|| ModelSelectionMode::manual(provider.clone(), model.clone(), None));
1723        let thread_id = uuid::Uuid::new_v4().to_string();
1724        let runner_binding = match req.runner {
1725            Some(selection) => Some(
1726                self.resolve_thread_runner_binding(&thread_id, selection)
1727                    .await?,
1728            ),
1729            // No explicit per-thread selection: if a runtime-level destination
1730            // is active and its provider advertises a default workspace, bind
1731            // the new thread so its coding tools route into the runner. This is
1732            // what makes a TUI/config-selected runner (e.g. Blaxel) actually
1733            // execute tools in the sandbox instead of locally. Providers that
1734            // return no default workspace stay unbound (legacy local-tools).
1735            None => {
1736                self.synthesize_runtime_runner_binding(
1737                    &thread_id,
1738                    cfg.remote_runner_destination.clone(),
1739                )
1740                .await?
1741            }
1742        };
1743        let runner_destination = runner_binding
1744            .as_ref()
1745            .map(|binding| binding.destination.clone())
1746            .or_else(|| cfg.remote_runner_destination.clone());
1747        let metadata = ThreadMetadata {
1748            thread_id,
1749            title: req.title,
1750            workspace,
1751            workspace_id: req.workspace_id,
1752            root_id: req.root_id,
1753            provider: Some(provider),
1754            model: Some(model),
1755            selection_mode: Some(selection_mode),
1756            tool_allowlist: req.tool_allowlist,
1757            developer_instructions: req.developer_instructions,
1758            external_tools: req.external_tools,
1759            runner_destination,
1760            runner_state: None,
1761            runner_binding,
1762            created_at: now,
1763            updated_at: now,
1764            message_count: 0,
1765            usage: None,
1766            parent_thread_id: None,
1767            forked_from_turn_id: None,
1768            workspace_fork: None,
1769        };
1770
1771        let metadata = if let Some(store) = &self.thread_store {
1772            store.create_thread(metadata).await?
1773        } else {
1774            metadata
1775        };
1776        self.emit(RoderEvent::ThreadCreated(ThreadCreated {
1777            thread_id: metadata.thread_id.clone(),
1778            timestamp: OffsetDateTime::now_utc(),
1779        }))
1780        .await;
1781        Ok(metadata)
1782    }
1783
1784    pub async fn list_threads(&self) -> anyhow::Result<Vec<ThreadMetadata>> {
1785        if let Some(store) = &self.thread_store {
1786            return store.list_threads().await;
1787        }
1788        Ok(Vec::new())
1789    }
1790
1791    pub async fn list_threads_page(
1792        &self,
1793        options: roder_api::thread::ThreadListOptions,
1794    ) -> anyhow::Result<roder_api::thread::ThreadListPage> {
1795        if let Some(store) = &self.thread_store {
1796            return store.list_threads_page(options).await;
1797        }
1798        Ok(roder_api::thread::ThreadListPage::default())
1799    }
1800
1801    pub async fn load_thread_metadata(
1802        &self,
1803        thread_id: &str,
1804    ) -> anyhow::Result<Option<roder_api::thread::ThreadMetadata>> {
1805        if let Some(store) = &self.thread_store {
1806            return store.load_thread_metadata(&thread_id.to_string()).await;
1807        }
1808        Ok(None)
1809    }
1810
1811    pub async fn archive_thread(&self, thread_id: &str) -> anyhow::Result<bool> {
1812        let archived = if let Some(store) = &self.thread_store {
1813            store.archive_thread(&thread_id.to_string()).await?
1814        } else {
1815            false
1816        };
1817        if archived {
1818            let workspace = self.config.read().await.workspace.clone();
1819            crate::hooks::run_lifecycle(
1820                self,
1821                &thread_id.to_string(),
1822                &SESSION_HOOK_TURN_ID.to_string(),
1823                workspace.as_deref(),
1824                "SessionEnd",
1825                Some("clear"),
1826                serde_json::json!({"reason": "archive"}),
1827            )
1828            .await;
1829            self.thread_item_cache
1830                .lock()
1831                .await
1832                .remove_thread(&thread_id.to_string());
1833            self.evict_runner_session(&thread_id.to_string()).await;
1834        }
1835        Ok(archived)
1836    }
1837
1838    pub async fn start_team(&self, req: TeamStartRequest) -> anyhow::Result<TeamState> {
1839        let cfg = self.config.read().await.clone();
1840        let workspace = self.workspace.display().to_string();
1841        let lead_thread_id = match req.lead_thread_id {
1842            Some(thread_id) => thread_id,
1843            None => {
1844                self.create_thread_with(CreateThreadRequest {
1845                    title: Some("Team lead".to_string()),
1846                    workspace: workspace.clone(),
1847                    workspace_id: None,
1848                    root_id: None,
1849                    provider: None,
1850                    model: None,
1851                    selection_mode: None,
1852                    tool_allowlist: Vec::new(),
1853                    developer_instructions: None,
1854                    external_tools: Vec::new(),
1855                    runner: None,
1856                })
1857                .await?
1858                .thread_id
1859            }
1860        };
1861        let team_id = uuid::Uuid::new_v4().to_string();
1862        let active_lead_turn_id = self.active_turn_for_thread(&lead_thread_id).await;
1863        let lead_selection = match active_lead_turn_id.as_ref() {
1864            Some(turn_id) => self
1865                .active_turn_selections
1866                .read()
1867                .await
1868                .get(turn_id)
1869                .cloned(),
1870            None => self.selection_mode_for_thread(&lead_thread_id).await?,
1871        };
1872        let lead_concrete_selection = lead_selection
1873            .as_ref()
1874            .map(ModelSelectionMode::concrete_selection);
1875        let mut lead = crate::teams::lead_member(
1876            lead_thread_id.clone(),
1877            lead_concrete_selection
1878                .as_ref()
1879                .map(|selection| selection.provider.clone())
1880                .or_else(|| Some(cfg.default_provider.clone())),
1881            lead_concrete_selection
1882                .as_ref()
1883                .map(|selection| selection.model.clone())
1884                .or_else(|| Some(cfg.default_model.clone())),
1885            cfg.policy_mode,
1886        );
1887        if let Some(turn_id) = active_lead_turn_id {
1888            lead.current_turn_id = Some(turn_id);
1889            lead.status = TeamMemberStatus::Running;
1890        }
1891        let mut members = vec![lead];
1892
1893        for (index, member) in req.members.into_iter().enumerate() {
1894            let thread = self
1895                .create_thread_with(CreateThreadRequest {
1896                    title: Some(member.name.clone()),
1897                    workspace: workspace.clone(),
1898                    workspace_id: None,
1899                    root_id: None,
1900                    provider: member.model_provider.clone(),
1901                    model: member.model.clone(),
1902                    selection_mode: None,
1903                    tool_allowlist: Vec::new(),
1904                    developer_instructions: None,
1905                    external_tools: Vec::new(),
1906                    runner: None,
1907                })
1908                .await?;
1909            let member_id = format!("member-{}", index + 1);
1910            let descriptor = crate::teams::teammate_member(
1911                member_id.clone(),
1912                member.name,
1913                thread.thread_id.clone(),
1914                member.model_provider.or(thread.provider),
1915                member.model.or(thread.model),
1916                cfg.policy_mode,
1917            );
1918            members.push(descriptor);
1919        }
1920
1921        let now = OffsetDateTime::now_utc();
1922        let team = self
1923            .teams
1924            .insert(TeamState {
1925                id: team_id.clone(),
1926                lead_thread_id: lead_thread_id.clone(),
1927                display_mode: req.display_mode,
1928                members,
1929                mailbox: Vec::new(),
1930                tasks: Vec::new(),
1931                created_at: now,
1932                updated_at: now,
1933            })
1934            .await?;
1935        self.emit(RoderEvent::TeamStarted(TeamStarted {
1936            team_id: team_id.clone(),
1937            lead_thread_id,
1938            display_mode: team.display_mode,
1939            members: team.members.clone(),
1940            tasks: team.tasks.clone(),
1941            timestamp: OffsetDateTime::now_utc(),
1942        }))
1943        .await;
1944        for member in team
1945            .members
1946            .iter()
1947            .filter(|member| member.role != roder_api::teams::TeamMemberRole::Lead)
1948        {
1949            self.emit(RoderEvent::TeamMemberStarted(TeamMemberStarted {
1950                team_id: team_id.clone(),
1951                member: member.clone(),
1952                timestamp: OffsetDateTime::now_utc(),
1953            }))
1954            .await;
1955        }
1956        Ok(team)
1957    }
1958
1959    pub async fn list_teams(&self) -> Vec<TeamState> {
1960        let active_thread_ids = self
1961            .active_turns
1962            .read()
1963            .await
1964            .values()
1965            .map(|handle| handle.thread_id.clone())
1966            .collect::<std::collections::HashSet<_>>();
1967        self.teams
1968            .list()
1969            .await
1970            .into_iter()
1971            .map(|team| runtime_local_team_view(team, &active_thread_ids))
1972            .collect()
1973    }
1974
1975    pub async fn read_team(&self, team_id: &str) -> Option<TeamState> {
1976        let active_thread_ids = self
1977            .active_turns
1978            .read()
1979            .await
1980            .values()
1981            .map(|handle| handle.thread_id.clone())
1982            .collect::<std::collections::HashSet<_>>();
1983        self.teams
1984            .get(team_id)
1985            .await
1986            .map(|team| runtime_local_team_view(team, &active_thread_ids))
1987    }
1988
1989    pub async fn start_team_member(
1990        &self,
1991        team_id: &str,
1992        req: TeamMemberStartRequest,
1993    ) -> anyhow::Result<TeamState> {
1994        self.start_team_member_with_selection(team_id, req, None)
1995            .await
1996    }
1997
1998    pub async fn message_team_member(
1999        self: &Arc<Self>,
2000        team_id: &str,
2001        member_id: &str,
2002        message: String,
2003    ) -> anyhow::Result<TurnId> {
2004        let team = self
2005            .read_team(team_id)
2006            .await
2007            .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2008        self.followup_team_member(&team.lead_thread_id, team_id, member_id, message)
2009            .await
2010    }
2011
2012    /// Queue a mailbox message without starting an idle agent. If the target is
2013    /// already running, the message is delivered at the next inference boundary.
2014    pub(crate) async fn queue_team_member_message(
2015        self: &Arc<Self>,
2016        caller_thread_id: &ThreadId,
2017        team_id: &str,
2018        member_id: &str,
2019        message: String,
2020    ) -> anyhow::Result<Option<TurnId>> {
2021        let _delivery_guard = self.agent_team_spawn_lock.lock().await;
2022        let team = self
2023            .read_team(team_id)
2024            .await
2025            .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2026        let member = team
2027            .members
2028            .iter()
2029            .find(|member| member.id == member_id)
2030            .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2031            .clone();
2032        if member.status == TeamMemberStatus::Closed {
2033            anyhow::bail!("subagent {} is closed", member.name);
2034        }
2035        let from_member_id = team
2036            .members
2037            .iter()
2038            .find(|candidate| candidate.thread_id == *caller_thread_id)
2039            .map(|candidate| candidate.id.clone());
2040        self.teams
2041            .append_mailbox_message(
2042                team_id,
2043                from_member_id,
2044                member_id.to_string(),
2045                TeamMailboxMessageKind::Message,
2046                message,
2047            )
2048            .await?;
2049        let Some(turn_id) = self.active_turn_for_thread(&member.thread_id).await else {
2050            return Ok(None);
2051        };
2052        self.deliver_pending_team_mailbox(team_id, member_id, member.thread_id, turn_id.clone())
2053            .await?;
2054        Ok(Some(turn_id))
2055    }
2056
2057    /// Deliver queued messages and start an idle agent, or steer its active turn.
2058    pub(crate) async fn followup_team_member(
2059        self: &Arc<Self>,
2060        caller_thread_id: &ThreadId,
2061        team_id: &str,
2062        member_id: &str,
2063        message: String,
2064    ) -> anyhow::Result<TurnId> {
2065        let _spawn_guard = self.agent_team_spawn_lock.lock().await;
2066        self.followup_team_member_locked(caller_thread_id, team_id, member_id, message)
2067            .await
2068    }
2069
2070    async fn followup_team_member_locked(
2071        self: &Arc<Self>,
2072        caller_thread_id: &ThreadId,
2073        team_id: &str,
2074        member_id: &str,
2075        message: String,
2076    ) -> anyhow::Result<TurnId> {
2077        let team = self
2078            .read_team(team_id)
2079            .await
2080            .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2081        let member = team
2082            .members
2083            .iter()
2084            .find(|member| member.id == member_id)
2085            .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2086            .clone();
2087        if member.status == TeamMemberStatus::Closed {
2088            anyhow::bail!("subagent {} is closed", member.name);
2089        }
2090        let from_member_id = team
2091            .members
2092            .iter()
2093            .find(|candidate| candidate.thread_id == *caller_thread_id)
2094            .map(|candidate| candidate.id.clone());
2095        self.teams
2096            .append_mailbox_message(
2097                team_id,
2098                from_member_id,
2099                member_id.to_string(),
2100                TeamMailboxMessageKind::NewTask,
2101                message,
2102            )
2103            .await?;
2104        if let Some(turn_id) = self.active_turn_for_thread(&member.thread_id).await {
2105            self.deliver_pending_team_mailbox(
2106                team_id,
2107                member_id,
2108                member.thread_id,
2109                turn_id.clone(),
2110            )
2111            .await?;
2112            return Ok(turn_id);
2113        }
2114
2115        self.ensure_codex_v2_team_capacity(&team, &member.id, None)
2116            .await?;
2117        let metadata = self.load_thread_metadata(&member.thread_id).await?;
2118        let inherited_context = self
2119            .team_member_turn_contexts
2120            .lock()
2121            .await
2122            .get(&member.thread_id)
2123            .cloned();
2124        let workspace = inherited_context
2125            .as_ref()
2126            .map(|context| context.workspace.clone())
2127            .or_else(|| metadata.as_ref().map(|metadata| metadata.workspace.clone()))
2128            .unwrap_or_else(|| self.workspace.display().to_string());
2129        let member_thread_id = member.thread_id;
2130        let turn_id = self
2131            .start_turn(StartTurnRequest {
2132                thread_id: member_thread_id.clone(),
2133                message: String::new(),
2134                images: Vec::new(),
2135                provider_override: member.model_provider,
2136                model_override: member.model,
2137                reasoning_override: metadata
2138                    .as_ref()
2139                    .and_then(|metadata| metadata.selection_mode.as_ref())
2140                    .and_then(ModelSelectionMode::reasoning)
2141                    .map(str::to_string),
2142                workspace,
2143                instructions: inherited_context
2144                    .as_ref()
2145                    .map(|context| context.instructions.clone())
2146                    .unwrap_or_else(crate::default_instructions),
2147                developer_context: inherited_context
2148                    .as_ref()
2149                    .and_then(|context| context.developer_context.clone()),
2150                task_ledger_required: false,
2151                service_tier_override: None,
2152            })
2153            .await?;
2154        Ok(turn_id)
2155    }
2156
2157    pub async fn set_team_member_policy_mode(
2158        &self,
2159        team_id: &str,
2160        member_id: &str,
2161        policy_mode: PolicyMode,
2162    ) -> anyhow::Result<TeamState> {
2163        self.teams
2164            .set_member_policy_mode(team_id, member_id, policy_mode)
2165            .await
2166    }
2167
2168    pub async fn interrupt_team_member(
2169        &self,
2170        team_id: &str,
2171        member_id: &str,
2172    ) -> anyhow::Result<Option<TurnId>> {
2173        let _interrupt_guard = self.agent_team_spawn_lock.lock().await;
2174        let team = self
2175            .read_team(team_id)
2176            .await
2177            .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2178        let member = team
2179            .members
2180            .iter()
2181            .find(|member| member.id == member_id)
2182            .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2183            .clone();
2184        if member.status != TeamMemberStatus::Running {
2185            return Ok(None);
2186        }
2187        let Some(turn_id) = member.current_turn_id.clone() else {
2188            return Ok(None);
2189        };
2190        if self
2191            .active_turn_for_thread(&member.thread_id)
2192            .await
2193            .as_ref()
2194            != Some(&turn_id)
2195        {
2196            return Ok(None);
2197        }
2198        self.interrupt_turn(member.thread_id.clone(), turn_id.clone())
2199            .await?;
2200        // Make queued mailbox work immediately eligible for the replacement turn while the
2201        // lifecycle lock still prevents a follow-up from starting. Delivery acknowledgements
2202        // are turn-owned, so a late acknowledgement from the aborted turn cannot steal a
2203        // reservation acquired by its replacement.
2204        self.teams
2205            .release_mailbox_reservations_for_turn(&turn_id)
2206            .await;
2207        self.teams
2208            .update_member(team_id, member_id, |member| {
2209                member.status = TeamMemberStatus::Interrupted;
2210                member.current_turn_id = None;
2211                member.terminal_error = None;
2212            })
2213            .await?;
2214        self.emit(RoderEvent::TeamMemberCompleted(TeamMemberCompleted {
2215            team_id: team_id.to_string(),
2216            member_id: member_id.to_string(),
2217            member_thread_id: member.thread_id,
2218            turn_id: Some(turn_id.clone()),
2219            status: TeamMemberStatus::Interrupted,
2220            final_message: member.final_message,
2221            error: None,
2222            timestamp: OffsetDateTime::now_utc(),
2223        }))
2224        .await;
2225        Ok(Some(turn_id))
2226    }
2227
2228    pub async fn close_team_member(
2229        &self,
2230        team_id: &str,
2231        member_id: &str,
2232    ) -> anyhow::Result<roder_api::teams::TeamMemberDescriptor> {
2233        let team = self
2234            .read_team(team_id)
2235            .await
2236            .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2237        let member = team
2238            .members
2239            .iter()
2240            .find(|member| member.id == member_id)
2241            .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2242            .clone();
2243        if member.role == roder_api::teams::TeamMemberRole::Lead {
2244            anyhow::bail!("team lead cannot be closed as a subagent");
2245        }
2246        let interrupted_turn_id = if member.status == TeamMemberStatus::Running {
2247            if let Some(turn_id) = member.current_turn_id.clone() {
2248                self.interrupt_turn(member.thread_id.clone(), turn_id.clone())
2249                    .await?;
2250                Some(turn_id)
2251            } else {
2252                None
2253            }
2254        } else {
2255            member.current_turn_id.clone()
2256        };
2257        let updated = self
2258            .teams
2259            .update_member(team_id, member_id, |member| {
2260                member.status = TeamMemberStatus::Closed;
2261                member.current_turn_id = None;
2262            })
2263            .await?;
2264        let closed = updated
2265            .members
2266            .iter()
2267            .find(|member| member.id == member_id)
2268            .cloned()
2269            .ok_or_else(|| anyhow::anyhow!("closed team member disappeared"))?;
2270        self.team_member_turn_contexts
2271            .lock()
2272            .await
2273            .remove(&closed.thread_id);
2274        self.emit(RoderEvent::TeamMemberCompleted(TeamMemberCompleted {
2275            team_id: team_id.to_string(),
2276            member_id: closed.id.clone(),
2277            member_thread_id: closed.thread_id.clone(),
2278            turn_id: interrupted_turn_id,
2279            status: TeamMemberStatus::Closed,
2280            final_message: closed.final_message.clone(),
2281            error: closed.terminal_error.clone(),
2282            timestamp: OffsetDateTime::now_utc(),
2283        }))
2284        .await;
2285        Ok(closed)
2286    }
2287
2288    pub async fn cleanup_team(&self, team_id: &str, force: bool) -> anyhow::Result<bool> {
2289        let Some(team) = self.read_team(team_id).await else {
2290            return Ok(false);
2291        };
2292        if !force
2293            && team
2294                .members
2295                .iter()
2296                .any(|member| member.status == TeamMemberStatus::Running)
2297        {
2298            anyhow::bail!("team {team_id:?} has active teammates; use forced cleanup");
2299        }
2300        if force {
2301            for member in team.members.iter().filter(|member| {
2302                member.role != roder_api::teams::TeamMemberRole::Lead
2303                    && member.status == TeamMemberStatus::Running
2304            }) {
2305                if let Some(turn_id) = member
2306                    .current_turn_id
2307                    .clone()
2308                    .or(self.active_turn_for_thread(&member.thread_id).await)
2309                {
2310                    let _ = self.interrupt_turn(member.thread_id.clone(), turn_id).await;
2311                }
2312            }
2313        }
2314        let removed = self.teams.remove(team_id).await?.is_some();
2315        if removed {
2316            let member_thread_ids = team
2317                .members
2318                .iter()
2319                .map(|member| member.thread_id.clone())
2320                .collect::<std::collections::HashSet<_>>();
2321            self.team_member_turn_contexts
2322                .lock()
2323                .await
2324                .retain(|thread_id, _| !member_thread_ids.contains(thread_id));
2325            self.emit(RoderEvent::TeamCleanupCompleted(TeamCleanupCompleted {
2326                team_id: team_id.to_string(),
2327                forced: force,
2328                timestamp: OffsetDateTime::now_utc(),
2329            }))
2330            .await;
2331        }
2332        Ok(removed)
2333    }
2334
2335    pub async fn effective_policy_mode_for_thread(&self, thread_id: &str) -> PolicyMode {
2336        if let Some(mode) = self.teams.policy_mode_for_thread(thread_id).await {
2337            return mode;
2338        }
2339        self.status().await.policy_mode
2340    }
2341
2342    async fn complete_team_member_turn_with_result(
2343        &self,
2344        thread_id: &ThreadId,
2345        turn_id: &TurnId,
2346        status: TeamMemberStatus,
2347        final_message: Option<String>,
2348        terminal_error: Option<String>,
2349    ) -> anyhow::Result<()> {
2350        let _delivery_guard = self.agent_team_spawn_lock.lock().await;
2351        let Some((team_id, member)) = self
2352            .teams
2353            .complete_member_turn(
2354                thread_id,
2355                turn_id,
2356                status,
2357                final_message.clone(),
2358                terminal_error.clone(),
2359            )
2360            .await?
2361        else {
2362            return Ok(());
2363        };
2364        if let Some(parent_thread_id) = member.parent_thread_id.as_ref()
2365            && let Some(team) = self.read_team(&team_id).await
2366            && let Some(parent) = team
2367                .members
2368                .iter()
2369                .find(|candidate| candidate.thread_id == *parent_thread_id)
2370        {
2371            let identity = member
2372                .agent_path
2373                .as_deref()
2374                .or(member.task_name.as_deref())
2375                .unwrap_or(&member.name);
2376            let mut report = format!("Agent {identity} finished with status {status:?}.");
2377            if let Some(message) = final_message.as_deref()
2378                && !message.trim().is_empty()
2379            {
2380                report.push_str("\n\nFinal result:\n");
2381                report.push_str(message);
2382            }
2383            if let Some(error) = terminal_error.as_deref()
2384                && !error.trim().is_empty()
2385            {
2386                report.push_str("\n\nTerminal error:\n");
2387                report.push_str(error);
2388            }
2389            let mailbox_appended = self
2390                .teams
2391                .append_mailbox_message(
2392                    &team_id,
2393                    Some(member.id.clone()),
2394                    parent.id.clone(),
2395                    TeamMailboxMessageKind::FinalAnswer,
2396                    report,
2397                )
2398                .await
2399                .is_ok();
2400            if mailbox_appended
2401                && let Some(parent_turn_id) = self.active_turn_for_thread(parent_thread_id).await
2402            {
2403                // The parent may finish between the active-turn lookup and enqueue. Its durable
2404                // mailbox entry remains pending for the next turn, while terminal observers must
2405                // still receive TeamMemberCompleted for this child.
2406                let _ = self
2407                    .deliver_pending_team_mailbox(
2408                        &team_id,
2409                        &parent.id,
2410                        parent_thread_id.clone(),
2411                        parent_turn_id,
2412                    )
2413                    .await;
2414            }
2415        }
2416        self.emit(RoderEvent::TeamMemberCompleted(TeamMemberCompleted {
2417            team_id,
2418            member_id: member.id,
2419            member_thread_id: member.thread_id,
2420            turn_id: Some(turn_id.clone()),
2421            status,
2422            final_message,
2423            error: terminal_error,
2424            timestamp: OffsetDateTime::now_utc(),
2425        }))
2426        .await;
2427        Ok(())
2428    }
2429
2430    /// Stops accepting new turns, requests interruption for every active turn,
2431    /// and waits for their runtime tasks to reach terminal cleanup up to
2432    /// `timeout`. Repeated calls are safe and continue draining the same set.
2433    pub async fn drain_active_turns(&self, timeout: std::time::Duration) -> RuntimeDrainOutcome {
2434        let started_at = tokio::time::Instant::now();
2435        let persistence_failures_before =
2436            self.lifecycle_persistence_failures.load(Ordering::Acquire);
2437        let turn_admission = self.turn_admission.lock().await;
2438        self.accepting_turns.store(false, Ordering::Release);
2439        let active = self
2440            .active_turns
2441            .read()
2442            .await
2443            .iter()
2444            .map(|(turn_id, handle)| (turn_id.clone(), handle.thread_id.clone()))
2445            .collect::<Vec<_>>();
2446        let draining = self
2447            .turn_drains
2448            .read()
2449            .await
2450            .iter()
2451            .map(|(turn_id, drain)| (turn_id.clone(), drain.thread_id.clone()))
2452            .collect::<Vec<_>>();
2453        drop(turn_admission);
2454
2455        let mut interrupted_turns = std::collections::BTreeMap::new();
2456        for (turn_id, thread_id) in active.iter().chain(draining.iter()) {
2457            interrupted_turns.insert(turn_id.clone(), thread_id.clone());
2458        }
2459        let interrupted_turn_ids = interrupted_turns.keys().cloned().collect::<Vec<_>>();
2460        for (turn_id, thread_id) in active {
2461            let _ = self
2462                .interrupt_turn_with_reason(thread_id, turn_id, TurnLifecycleReason::Shutdown)
2463                .await;
2464        }
2465
2466        let wait_for_empty = async {
2467            loop {
2468                let notified = self.active_turns_changed.notified();
2469                if self.active_turns.read().await.is_empty()
2470                    && self.turn_drains.read().await.is_empty()
2471                {
2472                    return;
2473                }
2474                notified.await;
2475            }
2476        };
2477        let outcome = if tokio::time::timeout(timeout, wait_for_empty).await.is_ok() {
2478            if self.lifecycle_persistence_failures.load(Ordering::Acquire)
2479                > persistence_failures_before
2480            {
2481                RuntimeDrainOutcome::PersistenceFailed {
2482                    interrupted_turn_ids,
2483                    remaining_turn_ids: Vec::new(),
2484                }
2485            } else {
2486                RuntimeDrainOutcome::Clean {
2487                    interrupted_turn_ids,
2488                }
2489            }
2490        } else {
2491            let active_remaining = self
2492                .active_turns
2493                .read()
2494                .await
2495                .iter()
2496                .map(|(turn_id, handle)| (turn_id.clone(), handle.drain.clone()))
2497                .collect::<Vec<_>>();
2498            let draining_remaining = self
2499                .turn_drains
2500                .read()
2501                .await
2502                .iter()
2503                .map(|(turn_id, drain)| (turn_id.clone(), drain.clone()))
2504                .collect::<Vec<_>>();
2505            let remaining = active_remaining
2506                .into_iter()
2507                .chain(draining_remaining)
2508                .collect::<std::collections::BTreeMap<_, _>>();
2509            let remaining_turn_ids = remaining.keys().cloned().collect::<Vec<_>>();
2510            for turn_id in &remaining_turn_ids {
2511                let handle = remaining
2512                    .get(turn_id)
2513                    .expect("remaining turn ID must have a drain handle");
2514                self.record_turn_lifecycle(
2515                    handle.thread_id.clone(),
2516                    turn_id.clone(),
2517                    TurnLifecycleState::InterruptRequested,
2518                    TurnCleanupState::TimedOut,
2519                    Some(TurnLifecycleReason::Shutdown),
2520                )
2521                .await;
2522            }
2523            if self.lifecycle_persistence_failures.load(Ordering::Acquire)
2524                > persistence_failures_before
2525            {
2526                RuntimeDrainOutcome::PersistenceFailed {
2527                    interrupted_turn_ids,
2528                    remaining_turn_ids,
2529                }
2530            } else {
2531                RuntimeDrainOutcome::DeadlineExceeded {
2532                    interrupted_turn_ids,
2533                    remaining_turn_ids,
2534                }
2535            }
2536        };
2537        self.record_lifecycle_drain_outcome(started_at, &outcome);
2538        outcome
2539    }
2540
2541    /// Enables turn submission after a prior drain attempt. This is intended
2542    /// for embedders that keep a runtime alive after a bounded drain timeout.
2543    pub fn resume_accepting_turns(&self) {
2544        self.accepting_turns.store(true, Ordering::Release);
2545    }
2546
2547    pub async fn turn_lifecycle_snapshot(
2548        &self,
2549        thread_id: &ThreadId,
2550    ) -> anyhow::Result<TurnLifecycleSnapshot> {
2551        let Some(store) = &self.thread_store else {
2552            return Ok(TurnLifecycleSnapshot::default());
2553        };
2554        let extension_states = store.load_extension_states(thread_id).await?;
2555        Ok(turn_lifecycle_snapshot(&extension_states))
2556    }
2557
2558    pub async fn load_thread(
2559        &self,
2560        thread_id: &ThreadId,
2561    ) -> anyhow::Result<Option<ThreadSnapshot>> {
2562        let loaded = if let Some(store) = &self.thread_store {
2563            store.load_thread(thread_id).await?
2564        } else {
2565            None
2566        };
2567        if let Some(snapshot) = loaded.as_ref() {
2568            let live_turn_ids = self
2569                .active_turns
2570                .read()
2571                .await
2572                .keys()
2573                .cloned()
2574                .chain(self.turn_drains.read().await.keys().cloned())
2575                .collect::<std::collections::HashSet<_>>();
2576            let lifecycle = turn_lifecycle_snapshot(&snapshot.extension_states);
2577            for record in lifecycle.records.into_iter().filter(|record| {
2578                record.state.requires_recovery() && !live_turn_ids.contains(&record.turn_id)
2579            }) {
2580                self.lifecycle_restart_reconciliations
2581                    .fetch_add(1, Ordering::AcqRel);
2582                self.record_turn_lifecycle(
2583                    record.thread_id,
2584                    record.turn_id,
2585                    TurnLifecycleState::RecoveryNeeded,
2586                    TurnCleanupState::Unknown,
2587                    Some(TurnLifecycleReason::RuntimeRestart),
2588                )
2589                .await;
2590            }
2591            self.emit(RoderEvent::ThreadLoaded(ThreadLoaded {
2592                thread_id: thread_id.clone(),
2593                timestamp: OffsetDateTime::now_utc(),
2594            }))
2595            .await;
2596        }
2597        Ok(loaded)
2598    }
2599
2600    pub async fn workspace_for_thread(&self, thread_id: &ThreadId) -> anyhow::Result<String> {
2601        if let Some(store) = &self.thread_store {
2602            let snapshot = store
2603                .load_thread(thread_id)
2604                .await?
2605                .ok_or_else(|| anyhow::anyhow!("thread not found: {thread_id}"));
2606            match snapshot {
2607                Ok(snapshot) => {
2608                    if let Some(metadata) = snapshot.metadata {
2609                        // Fork-backed threads fail closed before any write
2610                        // when their workspace was removed out-of-band.
2611                        if let Some(fork) = &metadata.workspace_fork
2612                            && fork.status == roder_api::forks::ForkStatus::Active
2613                            && !std::path::Path::new(&metadata.workspace).is_dir()
2614                        {
2615                            anyhow::bail!(
2616                                "workspace fork {} is missing its workspace at {}; restore it or \
2617                                 remove the fork before running turns in this thread",
2618                                fork.id,
2619                                metadata.workspace
2620                            );
2621                        }
2622                        return Ok(metadata.workspace);
2623                    }
2624                    eprintln!(
2625                        "thread metadata missing while resolving workspace for {thread_id}; falling back to runtime workspace"
2626                    );
2627                }
2628                Err(err) => {
2629                    eprintln!(
2630                        "thread missing while resolving workspace for {thread_id}: {err}; falling back to runtime workspace"
2631                    );
2632                }
2633            }
2634        }
2635        Ok(self.workspace.display().to_string())
2636    }
2637
2638    pub(crate) async fn selection_mode_for_thread(
2639        &self,
2640        thread_id: &ThreadId,
2641    ) -> anyhow::Result<Option<ModelSelectionMode>> {
2642        let Some(store) = &self.thread_store else {
2643            return Ok(None);
2644        };
2645        Ok(store
2646            .load_thread(thread_id)
2647            .await?
2648            .and_then(|snapshot| snapshot.metadata)
2649            .and_then(|metadata| {
2650                metadata
2651                    .selection_mode
2652                    .or_else(|| match (metadata.provider, metadata.model) {
2653                        (Some(provider), Some(model)) => {
2654                            Some(ModelSelectionMode::manual(provider, model, None))
2655                        }
2656                        _ => None,
2657                    })
2658            }))
2659    }
2660
2661    /// Concrete provider/model for configured-role subagents (`task`,
2662    /// `agent_swarm`). Prefers the live parent turn selection so children stay
2663    /// on SuperGrok/Grok (or whatever the lead is using) instead of process
2664    /// startup defaults such as openai/gpt-5.5.
2665    pub(crate) async fn parent_model_selection_for_subagents(
2666        &self,
2667        thread_id: &ThreadId,
2668        turn_id: &TurnId,
2669    ) -> Option<roder_api::inference::ModelSelection> {
2670        if let Some(selection) = self
2671            .active_turn_selections
2672            .read()
2673            .await
2674            .get(turn_id)
2675            .cloned()
2676        {
2677            return Some(selection.concrete_selection());
2678        }
2679        if let Ok(Some(selection)) = self.selection_mode_for_thread(thread_id).await {
2680            return Some(selection.concrete_selection());
2681        }
2682        let cfg = self.status().await;
2683        if cfg.default_provider.trim().is_empty() || cfg.default_model.trim().is_empty() {
2684            return None;
2685        }
2686        Some(roder_api::inference::ModelSelection {
2687            provider: cfg.default_provider,
2688            model: cfg.default_model,
2689        })
2690    }
2691
2692    /// Per-thread tool allowlist, developer instructions, and external tools persisted at thread creation.
2693    pub(crate) async fn thread_turn_overrides(
2694        &self,
2695        thread_id: &ThreadId,
2696    ) -> anyhow::Result<ThreadTurnOverrides> {
2697        let Some(store) = &self.thread_store else {
2698            return Ok(ThreadTurnOverrides::default());
2699        };
2700        Ok(store
2701            .load_thread_metadata(thread_id)
2702            .await?
2703            .map(|metadata| ThreadTurnOverrides {
2704                tool_allowlist: metadata.tool_allowlist,
2705                developer_instructions: metadata.developer_instructions,
2706                external_tools: metadata.external_tools,
2707            })
2708            .unwrap_or_default())
2709    }
2710
2711    pub async fn set_thread_selection_mode(
2712        &self,
2713        thread_id: &ThreadId,
2714        selection_mode: ModelSelectionMode,
2715    ) -> anyhow::Result<()> {
2716        let Some(store) = &self.thread_store else {
2717            return Ok(());
2718        };
2719        let Some(snapshot) = store.load_thread(thread_id).await? else {
2720            anyhow::bail!("thread not found: {thread_id}");
2721        };
2722        let Some(mut metadata) = snapshot.metadata else {
2723            return Ok(());
2724        };
2725        let concrete = selection_mode.concrete_selection();
2726        metadata.provider = Some(concrete.provider);
2727        metadata.model = Some(concrete.model);
2728        metadata.selection_mode = Some(selection_mode);
2729        metadata.updated_at = OffsetDateTime::now_utc();
2730        store.update_thread_metadata(metadata).await?;
2731        Ok(())
2732    }
2733
2734    async fn runner_session_for_thread(
2735        &self,
2736        thread_id: &ThreadId,
2737    ) -> anyhow::Result<Option<(RunnerDestination, Arc<dyn RemoteRunnerSession>)>> {
2738        let metadata = if let Some(store) = &self.thread_store {
2739            store.load_thread_metadata(thread_id).await?
2740        } else {
2741            None
2742        };
2743        // An explicit per-thread binding wins over the runtime-level destination.
2744        let destination = metadata
2745            .as_ref()
2746            .and_then(|metadata| metadata.runner_binding.as_ref())
2747            .map(|binding| binding.destination.clone())
2748            .or(self.config.read().await.remote_runner_destination.clone());
2749        let Some(destination) = destination else {
2750            self.evict_runner_session(thread_id).await;
2751            return Ok(None);
2752        };
2753        let provider = self
2754            .registry
2755            .remote_runner_providers
2756            .iter()
2757            .find(|provider| provider.id() == destination.provider_id)
2758            .cloned()
2759            .ok_or_else(|| {
2760                anyhow::anyhow!(
2761                    "remote runner provider {:?} is not installed",
2762                    destination.provider_id
2763                )
2764            })?;
2765        let persisted_state = metadata.and_then(|metadata| metadata.runner_state);
2766        let slot = {
2767            let mut sessions = self.runner_sessions.lock().await;
2768            match sessions.get(thread_id) {
2769                Some(slot) if slot.matches(&destination) => slot.clone(),
2770                _ => {
2771                    let slot = Arc::new(RunnerSessionSlot::empty(&destination));
2772                    sessions.insert(thread_id.clone(), slot.clone());
2773                    slot
2774                }
2775            }
2776        };
2777
2778        loop {
2779            let initialized = slot.initialized.notified();
2780            let mut state = slot.state.lock().await;
2781            match &*state {
2782                RunnerSessionSlotState::Ready(cached) => {
2783                    return Ok(Some((cached.destination.clone(), cached.session.clone())));
2784                }
2785                RunnerSessionSlotState::Failed(error) => {
2786                    return Err(anyhow::anyhow!(error.to_string()));
2787                }
2788                RunnerSessionSlotState::Initializing => {
2789                    drop(state);
2790                    initialized.await;
2791                }
2792                RunnerSessionSlotState::Empty => {
2793                    *state = RunnerSessionSlotState::Initializing;
2794                    drop(state);
2795                    self.spawn_runner_session_initialization(
2796                        thread_id.clone(),
2797                        destination.clone(),
2798                        provider.clone(),
2799                        persisted_state.clone(),
2800                        slot.clone(),
2801                    );
2802                }
2803            }
2804        }
2805    }
2806
2807    fn spawn_runner_session_initialization(
2808        &self,
2809        thread_id: ThreadId,
2810        destination: RunnerDestination,
2811        provider: Arc<dyn RemoteRunnerProvider>,
2812        persisted_state: Option<RunnerSessionState>,
2813        slot: Arc<RunnerSessionSlot>,
2814    ) {
2815        let thread_store = self.thread_store.clone();
2816        let runner_sessions = self.runner_sessions.clone();
2817        // The task intentionally outlives the turn that first requested the
2818        // workspace. Interrupting that turn must not strand the slot in
2819        // `Initializing` and deadlock every later tool or lifecycle call.
2820        drop(tokio::spawn(async move {
2821            let initialized = async {
2822                let session = if let Some(state) = persisted_state
2823                    && state.provider_id == destination.provider_id
2824                    && state.destination_id == destination.id
2825                {
2826                    match provider.resume_session(state).await {
2827                        Ok(session) => session,
2828                        Err(_) => provider.create_session(destination.clone()).await?,
2829                    }
2830                } else {
2831                    provider.create_session(destination.clone()).await?
2832                };
2833                let resolved = (destination.clone(), session.clone());
2834                // Persist before releasing the singleflight or dispatching
2835                // the first tool. A later process can rejoin even if the turn
2836                // that requested initialization has already been interrupted.
2837                Self::persist_runner_state_in_store(
2838                    thread_store.as_ref(),
2839                    &thread_id,
2840                    Some(&resolved),
2841                )
2842                .await?;
2843                Ok::<_, anyhow::Error>(CachedRunnerSession {
2844                    destination,
2845                    session,
2846                })
2847            }
2848            .await;
2849
2850            match initialized {
2851                Ok(cached) => {
2852                    *slot.state.lock().await = RunnerSessionSlotState::Ready(cached);
2853                    slot.initialized.notify_waiters();
2854                }
2855                Err(error) => {
2856                    let message: Arc<str> = Arc::from(format!("{error:#}"));
2857                    *slot.state.lock().await = RunnerSessionSlotState::Failed(message);
2858                    slot.initialized.notify_waiters();
2859                    let mut sessions = runner_sessions.lock().await;
2860                    let is_current = sessions
2861                        .get(&thread_id)
2862                        .is_some_and(|current| Arc::ptr_eq(current, &slot));
2863                    if is_current {
2864                        sessions.remove(&thread_id);
2865                    }
2866                }
2867            }
2868        }));
2869    }
2870
2871    async fn evict_runner_session(&self, thread_id: &ThreadId) {
2872        self.runner_sessions.lock().await.remove(thread_id);
2873    }
2874
2875    async fn cache_runner_session(
2876        &self,
2877        thread_id: &ThreadId,
2878        destination: RunnerDestination,
2879        session: Arc<dyn RemoteRunnerSession>,
2880    ) {
2881        let cached = CachedRunnerSession {
2882            destination,
2883            session,
2884        };
2885        self.runner_sessions.lock().await.insert(
2886            thread_id.clone(),
2887            Arc::new(RunnerSessionSlot::ready(cached)),
2888        );
2889    }
2890
2891    /// Read a thread's explicit runner binding without creating or resuming a
2892    /// session. Tool routing uses this before local previews so runner-backed
2893    /// and fail-closed hosted calls never inspect the host workspace.
2894    pub(crate) async fn runner_binding_for_thread(
2895        &self,
2896        thread_id: &ThreadId,
2897    ) -> anyhow::Result<Option<ThreadRunnerBinding>> {
2898        let Some(store) = &self.thread_store else {
2899            return Ok(None);
2900        };
2901        Ok(store
2902            .load_thread_metadata(thread_id)
2903            .await?
2904            .and_then(|metadata| metadata.runner_binding))
2905    }
2906
2907    /// Resolve a previously-read binding into a live remote workspace. The
2908    /// session remains lazy: callers invoke this only after policy approval.
2909    pub(crate) async fn remote_workspace_for_binding(
2910        &self,
2911        thread_id: &ThreadId,
2912        binding: ThreadRunnerBinding,
2913    ) -> anyhow::Result<Arc<RemoteWorkspace>> {
2914        let session = self
2915            .runner_session_for_thread(thread_id)
2916            .await?
2917            .map(|(_, session)| session)
2918            .ok_or_else(|| {
2919                anyhow::anyhow!("runner-bound thread {thread_id} has no runner session")
2920            })?;
2921        Ok(Arc::new(RemoteWorkspace {
2922            session,
2923            root: binding.workspace,
2924            read_roots: binding.read_roots,
2925        }))
2926    }
2927
2928    pub(crate) async fn runner_tool_execution_lock(
2929        &self,
2930        session: &dyn RemoteRunnerSession,
2931    ) -> Arc<RunnerToolExecutionMutex> {
2932        let state = session.state();
2933        let key = (state.provider_id, state.session_id);
2934        let mut locks = self.runner_tool_execution_locks.lock().await;
2935        if let Some(lock) = locks.get(&key).and_then(Weak::upgrade) {
2936            return lock;
2937        }
2938
2939        locks.retain(|_, lock| lock.strong_count() > 0);
2940        let lock = Arc::new(Mutex::new(()));
2941        locks.insert(key, Arc::downgrade(&lock));
2942        lock
2943    }
2944
2945    async fn persist_runner_state(
2946        &self,
2947        thread_id: &ThreadId,
2948        runner: Option<&(RunnerDestination, Arc<dyn RemoteRunnerSession>)>,
2949    ) -> anyhow::Result<()> {
2950        Self::persist_runner_state_in_store(self.thread_store.as_ref(), thread_id, runner).await
2951    }
2952
2953    async fn persist_runner_state_in_store(
2954        store: Option<&Arc<dyn ThreadStore>>,
2955        thread_id: &ThreadId,
2956        runner: Option<&(RunnerDestination, Arc<dyn RemoteRunnerSession>)>,
2957    ) -> anyhow::Result<()> {
2958        let Some((destination, session)) = runner else {
2959            return Ok(());
2960        };
2961        let Some(store) = store else {
2962            return Ok(());
2963        };
2964        let Some(snapshot) = store.load_thread(thread_id).await? else {
2965            return Ok(());
2966        };
2967        let Some(mut metadata) = snapshot.metadata else {
2968            return Ok(());
2969        };
2970        metadata.runner_destination = Some(destination.clone());
2971        metadata.runner_state = Some(session.state());
2972        metadata.updated_at = OffsetDateTime::now_utc();
2973        store.update_thread_metadata(metadata).await?;
2974        Ok(())
2975    }
2976
2977    fn remote_runner_provider_by_id(
2978        &self,
2979        provider_id: &str,
2980    ) -> Option<Arc<dyn RemoteRunnerProvider>> {
2981        self.registry
2982            .remote_runner_providers
2983            .iter()
2984            .find(|provider| provider.id() == provider_id)
2985            .cloned()
2986    }
2987
2988    /// Resolve the live runner session for a thread along with its provider,
2989    /// failing clearly when the thread is not runner-bound.
2990    async fn thread_runner_session(
2991        &self,
2992        thread_id: &ThreadId,
2993    ) -> anyhow::Result<(
2994        RunnerDestination,
2995        Arc<dyn RemoteRunnerProvider>,
2996        Arc<dyn RemoteRunnerSession>,
2997    )> {
2998        let Some((destination, session)) = self.runner_session_for_thread(thread_id).await? else {
2999            anyhow::bail!("thread {thread_id} is not bound to a remote runner");
3000        };
3001        let provider = self
3002            .remote_runner_provider_by_id(&destination.provider_id)
3003            .ok_or_else(|| {
3004                anyhow::anyhow!(
3005                    "remote runner provider {:?} is not installed",
3006                    destination.provider_id
3007                )
3008            })?;
3009        Ok((destination, provider, session))
3010    }
3011
3012    /// Pause a runner-bound thread's session toward standby and persist the
3013    /// post-pause state. Errors if the provider is not pausable.
3014    pub async fn pause_thread_runner(
3015        &self,
3016        thread_id: &ThreadId,
3017    ) -> anyhow::Result<RunnerSessionState> {
3018        let (destination, provider, session) = self.thread_runner_session(thread_id).await?;
3019        anyhow::ensure!(
3020            provider.capabilities().pausable,
3021            "remote runner provider {:?} does not support pausing",
3022            destination.provider_id
3023        );
3024        let state = session.pause().await?;
3025        self.persist_runner_state(thread_id, Some(&(destination, session)))
3026            .await?;
3027        Ok(state)
3028    }
3029
3030    /// Resume (wake) a runner-bound thread's paused session and persist state.
3031    pub async fn resume_thread_runner(
3032        &self,
3033        thread_id: &ThreadId,
3034    ) -> anyhow::Result<RunnerSessionState> {
3035        let (destination, _provider, session) = self.thread_runner_session(thread_id).await?;
3036        let state = session.resume().await?;
3037        self.persist_runner_state(thread_id, Some(&(destination, session)))
3038            .await?;
3039        Ok(state)
3040    }
3041
3042    /// Detach a runner-bound thread's session: persist the durable, rejoinable
3043    /// state and leave the remote sandbox alive. Errors if not detachable.
3044    pub async fn detach_thread_runner(
3045        &self,
3046        thread_id: &ThreadId,
3047    ) -> anyhow::Result<RunnerSessionState> {
3048        let (destination, provider, session) = self.thread_runner_session(thread_id).await?;
3049        anyhow::ensure!(
3050            provider.capabilities().detachable,
3051            "remote runner provider {:?} does not support detaching",
3052            destination.provider_id
3053        );
3054        let state = session.detach().await?;
3055        // Persist the detached state explicitly so a later turn or process
3056        // rejoins the same sandbox instead of provisioning a new one.
3057        if let Some(store) = &self.thread_store
3058            && let Some(snapshot) = store.load_thread(thread_id).await?
3059            && let Some(mut metadata) = snapshot.metadata
3060        {
3061            metadata.runner_destination = Some(destination);
3062            metadata.runner_state = Some(state.clone());
3063            metadata.updated_at = OffsetDateTime::now_utc();
3064            store.update_thread_metadata(metadata).await?;
3065        }
3066        self.evict_runner_session(thread_id).await;
3067        Ok(state)
3068    }
3069
3070    /// Rejoin a previously created sandbox from a thread's persisted runner
3071    /// state without provisioning a new one. An optional `sandbox` overrides the
3072    /// persisted sandbox name (recovery by name). Persists refreshed state.
3073    pub async fn rejoin_thread_runner(
3074        &self,
3075        thread_id: &ThreadId,
3076        sandbox: Option<String>,
3077    ) -> anyhow::Result<RunnerSessionState> {
3078        let store = self
3079            .thread_store
3080            .as_ref()
3081            .ok_or_else(|| anyhow::anyhow!("thread store is required to rejoin a runner"))?;
3082        let metadata = store
3083            .load_thread_metadata(thread_id)
3084            .await?
3085            .ok_or_else(|| anyhow::anyhow!("thread {thread_id} has no metadata"))?;
3086        let destination = metadata
3087            .runner_binding
3088            .as_ref()
3089            .map(|binding| binding.destination.clone())
3090            .or_else(|| metadata.runner_destination.clone())
3091            .ok_or_else(|| anyhow::anyhow!("thread {thread_id} is not bound to a remote runner"))?;
3092        let mut state = metadata
3093            .runner_state
3094            .clone()
3095            .ok_or_else(|| anyhow::anyhow!("thread {thread_id} has no persisted runner state"))?;
3096        if let Some(sandbox) = sandbox
3097            && let Some(object) = state.metadata.as_object_mut()
3098        {
3099            object.insert("sandbox_name".to_string(), serde_json::Value::from(sandbox));
3100        }
3101        let provider = self
3102            .remote_runner_provider_by_id(&destination.provider_id)
3103            .ok_or_else(|| {
3104                anyhow::anyhow!(
3105                    "remote runner provider {:?} is not installed",
3106                    destination.provider_id
3107                )
3108            })?;
3109        let session = provider.rejoin_session(state).await?;
3110        self.persist_runner_state(thread_id, Some(&(destination.clone(), session.clone())))
3111            .await?;
3112        self.cache_runner_session(thread_id, destination, session.clone())
3113            .await;
3114        Ok(session.state())
3115    }
3116
3117    pub(crate) async fn record_thread_usage_metadata(
3118        &self,
3119        thread_id: &ThreadId,
3120        usage: &TokenUsage,
3121    ) -> anyhow::Result<()> {
3122        if usage.is_empty() {
3123            return Ok(());
3124        }
3125        let Some(store) = &self.thread_store else {
3126            return Ok(());
3127        };
3128        let Some(snapshot) = store.load_thread(thread_id).await? else {
3129            return Ok(());
3130        };
3131        let Some(mut metadata) = snapshot.metadata else {
3132            return Ok(());
3133        };
3134        metadata
3135            .usage
3136            .get_or_insert_with(ThreadUsageMetadata::default)
3137            .add_token_usage(usage);
3138        metadata.updated_at = OffsetDateTime::now_utc();
3139        store.update_thread_metadata(metadata).await?;
3140        Ok(())
3141    }
3142
3143    pub(crate) async fn has_active_turn_for_thread(&self, thread_id: &ThreadId) -> bool {
3144        self.active_turns
3145            .read()
3146            .await
3147            .values()
3148            .any(|handle| &handle.thread_id == thread_id)
3149    }
3150
3151    async fn ensure_codex_v2_team_capacity(
3152        &self,
3153        team: &TeamState,
3154        resuming_member_id: &str,
3155        candidate_model: Option<&str>,
3156    ) -> anyhow::Result<()> {
3157        if !is_codex_v2_team(team)
3158            && !candidate_model
3159                .is_some_and(|model| model_supports_reasoning_effort(model, REASONING_ULTRA))
3160        {
3161            return Ok(());
3162        }
3163        let active_thread_ids = self
3164            .active_turns
3165            .read()
3166            .await
3167            .values()
3168            .map(|handle| handle.thread_id.clone())
3169            .collect::<std::collections::HashSet<_>>();
3170        let resident_threads = 1 + team
3171            .members
3172            .iter()
3173            .filter(|member| {
3174                member.role != roder_api::teams::TeamMemberRole::Lead
3175                    && member.id != resuming_member_id
3176                    && active_thread_ids.contains(&member.thread_id)
3177            })
3178            .count();
3179        anyhow::ensure!(
3180            resident_threads < codex_v2::CODEX_V2_MAX_RESIDENT_TEAM_THREADS,
3181            "agent thread limit reached for this Codex V2 team: maximum {} resident threads (the lead plus 3 running subagents)",
3182            codex_v2::CODEX_V2_MAX_RESIDENT_TEAM_THREADS
3183        );
3184        Ok(())
3185    }
3186
3187    /// Number of currently running turns across all threads. Hosted runtime
3188    /// pools use this to avoid evicting tenants with active work.
3189    pub async fn active_turn_count(&self) -> usize {
3190        self.active_turns.read().await.len()
3191    }
3192
3193    pub async fn active_turn_for_thread(&self, thread_id: &ThreadId) -> Option<TurnId> {
3194        self.active_turns
3195            .read()
3196            .await
3197            .iter()
3198            .find_map(|(turn_id, handle)| (&handle.thread_id == thread_id).then(|| turn_id.clone()))
3199    }
3200
3201    pub async fn thread_activity(&self, thread_id: &ThreadId) -> ThreadActivity {
3202        let active_turn_id = self.active_turn_for_thread(thread_id).await;
3203        let draining_turn_id = if active_turn_id.is_none() {
3204            self.turn_drains
3205                .read()
3206                .await
3207                .iter()
3208                .find_map(|(turn_id, drain)| {
3209                    (&drain.thread_id == thread_id).then(|| turn_id.clone())
3210                })
3211        } else {
3212            None
3213        };
3214        let Some(active_turn_id) = active_turn_id.or(draining_turn_id) else {
3215            return ThreadActivity::default();
3216        };
3217
3218        let mut active_flags = Vec::new();
3219        {
3220            let pending_approvals = self.pending_tool_approvals.lock().await;
3221            if pending_approvals
3222                .values()
3223                .any(|pending| &pending.thread_id == thread_id && pending.turn_id == active_turn_id)
3224            {
3225                active_flags.push("approvalRequired".to_string());
3226            }
3227        }
3228        {
3229            let pending_inputs = self.pending_user_inputs.lock().await;
3230            if pending_inputs
3231                .values()
3232                .any(|pending| &pending.thread_id == thread_id && pending.turn_id == active_turn_id)
3233            {
3234                active_flags.push("userInputRequired".to_string());
3235            }
3236        }
3237        {
3238            let pending_external = self.pending_external_tool_calls.lock().await;
3239            if pending_external
3240                .values()
3241                .any(|pending| &pending.thread_id == thread_id && pending.turn_id == active_turn_id)
3242            {
3243                active_flags.push("externalToolPending".to_string());
3244            }
3245        }
3246        let interrupting = self
3247            .active_turns
3248            .read()
3249            .await
3250            .get(&active_turn_id)
3251            .is_some_and(|handle| handle.drain.interrupt_requested.load(Ordering::Acquire))
3252            || self
3253                .turn_drains
3254                .read()
3255                .await
3256                .get(&active_turn_id)
3257                .is_some_and(|drain| drain.interrupt_requested.load(Ordering::Acquire));
3258        if interrupting {
3259            active_flags.push("interrupting".to_string());
3260        }
3261        if self.pending_plan_exit().await.is_some_and(|pending| {
3262            &pending.thread_id == thread_id && pending.turn_id == active_turn_id
3263        }) {
3264            active_flags.push("planExitRequired".to_string());
3265        }
3266
3267        ThreadActivity {
3268            active_turn_id: Some(active_turn_id),
3269            active_flags,
3270        }
3271    }
3272
3273    pub async fn interrupt_turn(&self, thread_id: ThreadId, turn_id: TurnId) -> anyhow::Result<()> {
3274        self.interrupt_turn_with_reason(thread_id, turn_id, TurnLifecycleReason::UserInterrupt)
3275            .await
3276    }
3277
3278    async fn interrupt_turn_with_reason(
3279        &self,
3280        thread_id: ThreadId,
3281        turn_id: TurnId,
3282        reason: TurnLifecycleReason,
3283    ) -> anyhow::Result<()> {
3284        let handle = self.active_turns.write().await.remove(&turn_id);
3285        if let Some(handle) = handle {
3286            if handle
3287                .drain
3288                .interrupt_requested
3289                .swap(true, Ordering::AcqRel)
3290            {
3291                return Ok(());
3292            }
3293            *handle.drain.interrupt_reason.lock().await = Some(reason);
3294            self.turn_drains
3295                .write()
3296                .await
3297                .insert(turn_id.clone(), handle.drain.clone());
3298            let ownership = self.provider_cleanup_ownership(&turn_id);
3299            self.record_turn_lifecycle_with_ownership(
3300                thread_id.clone(),
3301                turn_id.clone(),
3302                TurnLifecycleState::InterruptRequested,
3303                TurnCleanupState::Requested,
3304                Some(reason),
3305                ownership,
3306            )
3307            .await;
3308            handle.abort.abort();
3309            if reason == TurnLifecycleReason::UserInterrupt {
3310                self.goals
3311                    .finish_turn(
3312                        &thread_id,
3313                        &turn_id,
3314                        Some(roder_api::goals::ThreadGoalStatus::Paused),
3315                    )
3316                    .await?;
3317            }
3318            self.active_turns_changed.notify_waiters();
3319        }
3320        self.active_turn_selections.write().await.remove(&turn_id);
3321        self.cancel_pending_external_tool_calls_for_turn(&turn_id)
3322            .await;
3323        Ok(())
3324    }
3325
3326    pub async fn steer_turn(
3327        &self,
3328        thread_id: ThreadId,
3329        turn_id: TurnId,
3330        message: String,
3331        images: Vec<InputImage>,
3332    ) -> anyhow::Result<()> {
3333        self.enqueue_turn_steer(thread_id, turn_id, message, images, None)
3334            .await
3335    }
3336
3337    pub(crate) async fn has_pending_turn_steers(&self, turn_id: &TurnId) -> bool {
3338        let active = self.active_turns.read().await.get(turn_id).cloned();
3339        let Some(active) = active else {
3340            return false;
3341        };
3342        let has_pending = !active.steers.lock().await.is_empty();
3343        has_pending
3344    }
3345
3346    async fn steer_turn_with_mailbox_ack(
3347        &self,
3348        thread_id: ThreadId,
3349        turn_id: TurnId,
3350        message: String,
3351        message_ids: Vec<String>,
3352        team_id: TeamId,
3353    ) -> anyhow::Result<()> {
3354        self.enqueue_turn_steer(
3355            thread_id,
3356            turn_id,
3357            message,
3358            Vec::new(),
3359            Some(MailboxDeliveryAck {
3360                team_id,
3361                message_ids,
3362            }),
3363        )
3364        .await
3365    }
3366
3367    async fn deliver_pending_team_mailbox(
3368        &self,
3369        team_id: &str,
3370        member_id: &str,
3371        member_thread_id: ThreadId,
3372        turn_id: TurnId,
3373    ) -> anyhow::Result<()> {
3374        let pending = self
3375            .teams
3376            .reserve_pending_mailbox_messages(team_id, member_id, &turn_id)
3377            .await?;
3378        if pending.is_empty() {
3379            return Ok(());
3380        }
3381        let message_ids = pending
3382            .iter()
3383            .map(|message| message.id.clone())
3384            .collect::<Vec<_>>();
3385        let team = self
3386            .read_team(team_id)
3387            .await
3388            .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
3389        if let Err(error) = self
3390            .steer_turn_with_mailbox_ack(
3391                member_thread_id,
3392                turn_id.clone(),
3393                format_mailbox_messages(&team, &pending),
3394                message_ids.clone(),
3395                team_id.to_string(),
3396            )
3397            .await
3398        {
3399            self.teams
3400                .release_mailbox_reservations(&turn_id, &message_ids)
3401                .await;
3402            return Err(error);
3403        }
3404        Ok(())
3405    }
3406
3407    async fn enqueue_turn_steer(
3408        &self,
3409        thread_id: ThreadId,
3410        turn_id: TurnId,
3411        message: String,
3412        images: Vec<InputImage>,
3413        mailbox_ack: Option<MailboxDeliveryAck>,
3414    ) -> anyhow::Result<()> {
3415        let message = message.trim().to_string();
3416        if message.is_empty() && images.is_empty() {
3417            return Ok(());
3418        }
3419
3420        let Some(active) = self.active_turns.read().await.get(&turn_id).cloned() else {
3421            anyhow::bail!("no active turn to steer");
3422        };
3423        anyhow::ensure!(
3424            active.thread_id == thread_id,
3425            "turn does not belong to thread"
3426        );
3427        {
3428            let mut steers = active.steers.lock().await;
3429            steers.push(QueuedTurnSteer {
3430                message: UserMessage::with_images(message.clone(), images),
3431                mailbox_ack,
3432            });
3433            active
3434                .steer_changed
3435                .send_modify(|generation| *generation = generation.wrapping_add(1));
3436        }
3437        self.emit(RoderEvent::TurnSteered(TurnSteered {
3438            thread_id,
3439            turn_id,
3440            message,
3441            timestamp: OffsetDateTime::now_utc(),
3442        }))
3443        .await;
3444        Ok(())
3445    }
3446
3447    pub async fn tool_specs(&self) -> Vec<roder_api::tools::ToolSpec> {
3448        let cfg = self.config.read().await;
3449        let model_profile =
3450            model_profile_for_provider_model(&cfg, &cfg.default_provider, &cfg.default_model);
3451        self.filtered_tool_specs(
3452            &cfg,
3453            &cfg.default_model,
3454            &cfg.default_provider,
3455            model_profile.as_ref(),
3456            &[],
3457            &[],
3458        )
3459    }
3460
3461    pub fn subagent_definitions(&self) -> Vec<SubagentDefinition> {
3462        self.registry
3463            .subagent_dispatchers
3464            .iter()
3465            .flat_map(|dispatcher| dispatcher.definitions())
3466            .collect()
3467    }
3468
3469    async fn run_turn(
3470        self: &Arc<Self>,
3471        req: StartTurnRequest,
3472        turn_id: TurnId,
3473        initial_mailbox_ack: Option<MailboxDeliveryAck>,
3474        mut steering: tokio::sync::watch::Receiver<u64>,
3475        automatic_goal_turn: bool,
3476    ) -> anyhow::Result<TurnRunOutcome> {
3477        self.ensure_execution_authority()?;
3478        self.goals
3479            .begin_turn(&req.thread_id, &turn_id, automatic_goal_turn)
3480            .await?;
3481        let turn_started_at = OffsetDateTime::now_utc();
3482        self.emit(RoderEvent::TurnStarted(TurnStarted {
3483            thread_id: req.thread_id.clone(),
3484            turn_id: turn_id.clone(),
3485            runtime_profile: self.config.read().await.runtime_profile,
3486            timestamp: turn_started_at,
3487        }))
3488        .await;
3489        self.persist_turn_item(
3490            &req.thread_id,
3491            &turn_id,
3492            &TranscriptItem::UserMessage(UserMessage::with_images(
3493                req.message.clone(),
3494                req.images.clone(),
3495            )),
3496        )
3497        .await?;
3498        crate::hooks::run_lifecycle(
3499            self,
3500            &req.thread_id,
3501            &turn_id,
3502            Some(&req.workspace),
3503            "UserPromptSubmit",
3504            None,
3505            serde_json::json!({"prompt": req.message}),
3506        )
3507        .await;
3508        if let Some(ack) = initial_mailbox_ack {
3509            self.teams
3510                .mark_mailbox_messages_delivered(&ack.team_id, &turn_id, &ack.message_ids)
3511                .await?;
3512        }
3513
3514        let mut cfg = self.config.read().await.clone();
3515        let runtime_profile = cfg.runtime_profile;
3516        let turn_deadline = turn_deadline_for_config(&cfg);
3517        let deadline_finalization_reserve =
3518            crate::deadline_policy::finalization_reserve_seconds(cfg.turn_deadline_seconds);
3519        let selection_mode = self.selection_mode_for_thread(&req.thread_id).await?;
3520        let concrete_selection = selection_mode
3521            .as_ref()
3522            .map(ModelSelectionMode::concrete_selection);
3523        let default_provider = req
3524            .provider_override
3525            .clone()
3526            .or_else(|| {
3527                concrete_selection
3528                    .as_ref()
3529                    .map(|selection| selection.provider.clone())
3530            })
3531            .unwrap_or(cfg.default_provider.clone());
3532        let default_model = req
3533            .model_override
3534            .clone()
3535            .or_else(|| {
3536                concrete_selection
3537                    .as_ref()
3538                    .map(|selection| selection.model.clone())
3539            })
3540            .unwrap_or(cfg.default_model.clone());
3541        if let Some(reasoning) = req.reasoning_override.as_deref().or_else(|| {
3542            selection_mode
3543                .as_ref()
3544                .and_then(ModelSelectionMode::reasoning)
3545        }) {
3546            validate_reasoning_effort(&default_model, reasoning)?;
3547            cfg.reasoning = Some(reasoning.to_string());
3548        }
3549        let turn_has_concrete_model_override =
3550            req.provider_override.is_some() || req.model_override.is_some();
3551        let (turn_inference_router, turn_inference_router_profile) = match &selection_mode {
3552            Some(ModelSelectionMode::Auto {
3553                router_id, profile, ..
3554            }) if !turn_has_concrete_model_override => (
3555                RuntimeInferenceRouterConfig {
3556                    enabled: true,
3557                    router_id: Some(router_id.clone()),
3558                },
3559                profile.clone(),
3560            ),
3561            _ => (RuntimeInferenceRouterConfig::disabled(), None),
3562        };
3563        let mut provider = default_provider.clone();
3564        let mut model = default_model.clone();
3565        let mut model_profile = model_profile_for_provider_model(&cfg, &provider, &model);
3566        let workspace = req.workspace.clone();
3567        let compaction_generation_at_start = self.compaction_generation(&req.thread_id);
3568        let mut transcript = self.transcript_for_turn(&req, &turn_id).await?;
3569        let mut compacted_this_turn =
3570            self.compaction_generation(&req.thread_id) != compaction_generation_at_start;
3571        let agent_swarm_mode_active = self
3572            .effective_agent_swarm_mode_for_thread(&req.thread_id)
3573            .await;
3574        let ultra_mode_active = self.effective_ultra_mode_for_thread(&req.thread_id).await;
3575        let thread_overrides = self.thread_turn_overrides(&req.thread_id).await?;
3576        let mut final_assistant_text = String::new();
3577        let mut final_provider_metadata = None;
3578        let mut exhausted_tool_rounds = true;
3579        let mut verification_gate =
3580            VerificationGateState::new(req.message.clone(), runtime_profile);
3581        let mut speed_policy = SpeedPolicyState::default();
3582        let mut reliability = TurnReliabilityState::default();
3583        let mut turn_usage = TokenUsage::default();
3584        // Overwritten on every inference step's Completed event so only the
3585        // terminal step's stop reason survives (mid-turn tool_use steps must
3586        // not leak onto turn/completed).
3587        let mut turn_finish_reason: Option<String> = None;
3588        let mut deadline_finalization_requested = false;
3589        let mut deadline_scoreable_completion_requested = false;
3590        let mut task_ledger_completion_reminders = 0_u8;
3591        let mut task_ledger_scoreable_checkpoints = 0_u8;
3592        let mut empty_tool_call_nudges_used = 0_u32;
3593        let mut provider_stream_retry_attempts = 0_u32;
3594        let mut context_limit_recovery_attempts = 0_u32;
3595        let mut routing_candidates = None;
3596        let routing_transcript_start = transcript.len().saturating_sub(1);
3597        let mut routing_escalations = 0_u32;
3598        let mut model_switch_summary_selection = None::<ModelSelection>;
3599
3600        'tool_rounds: for round_index in 0..MAX_TOOL_ROUNDS_PER_TURN {
3601            if let Some(deadline) = turn_deadline
3602                && deadline_expired(deadline)
3603            {
3604                self.fail_turn_due_to_deadline(&req.thread_id, &turn_id, deadline, &transcript)
3605                    .await?;
3606                return Ok(TurnRunOutcome::Stopped);
3607            }
3608            let steers = self.drain_turn_steers(&turn_id, &mut steering).await;
3609            self.append_steers(&req, &turn_id, &mut transcript, steers)
3610                .await?;
3611            if runtime_profile == RuntimeProfile::Eval
3612                && let Some(remaining) = crate::deadline_policy::should_start_finalization(
3613                    turn_deadline,
3614                    deadline_finalization_reserve,
3615                    deadline_finalization_requested || deadline_scoreable_completion_requested,
3616                )
3617            {
3618                if req.task_ledger_required
3619                    && task_ledger_completion_reminders < TASK_LEDGER_COMPLETION_REMINDER_LIMIT
3620                    && let Some(prompt) = task_ledger_completion_prompt(&transcript)
3621                {
3622                    task_ledger_completion_reminders += 1;
3623                    deadline_scoreable_completion_requested = true;
3624                    let item = TranscriptItem::UserMessage(UserMessage::text(
3625                        task_ledger_deadline_completion_prompt(
3626                            remaining,
3627                            deadline_finalization_reserve,
3628                            &prompt,
3629                        ),
3630                    ));
3631                    self.persist_turn_item(&req.thread_id, &turn_id, &item)
3632                        .await?;
3633                    transcript.push(item);
3634                    continue 'tool_rounds;
3635                } else {
3636                    self.start_deadline_finalization(
3637                        &req.thread_id,
3638                        &turn_id,
3639                        &mut transcript,
3640                        remaining,
3641                    )
3642                    .await?;
3643                    deadline_finalization_requested = true;
3644                }
3645            }
3646            if runtime_profile == RuntimeProfile::Eval
3647                && req.task_ledger_required
3648                && !deadline_finalization_requested
3649                && task_ledger_scoreable_checkpoints < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
3650                && let Some(remaining) = deadline_remaining_seconds(turn_deadline)
3651                && remaining <= TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS
3652                && remaining > deadline_finalization_reserve
3653                && let Some(prompt) = task_ledger_completion_prompt(&transcript)
3654            {
3655                task_ledger_scoreable_checkpoints += 1;
3656                let item = TranscriptItem::UserMessage(UserMessage::text(
3657                    task_ledger_scoreable_checkpoint_prompt(remaining, &prompt),
3658                ));
3659                self.persist_turn_item(&req.thread_id, &turn_id, &item)
3660                    .await?;
3661                transcript.push(item);
3662                continue 'tool_rounds;
3663            }
3664            if turn_inference_router.is_active() && routing_candidates.is_none() {
3665                routing_candidates =
3666                    Some(collect_inference_routing_candidates(&self.registry).await);
3667            }
3668            let routing_tools_model = model.clone();
3669            let routing_tools_provider = provider.clone();
3670            let routing_tools = self.filtered_tool_specs(
3671                &cfg,
3672                &model,
3673                &provider,
3674                model_profile.as_ref(),
3675                &thread_overrides.tool_allowlist,
3676                &thread_overrides.external_tools,
3677            );
3678            let prior_failures =
3679                transcript_failure_count_since(&transcript, routing_transcript_start)
3680                    .max(reliability.tool_failure_count())
3681                    .saturating_add(provider_stream_retry_attempts);
3682            let routing_selection = route_inference_selection(
3683                &self.registry,
3684                &turn_inference_router,
3685                InferenceRoutingRequest {
3686                    thread_id: &req.thread_id,
3687                    turn_id: &turn_id,
3688                    round_index: round_index as u32,
3689                    runtime_profile,
3690                    phase: speed_policy.phase(),
3691                    profile: turn_inference_router_profile.as_deref(),
3692                    default_selection: ModelSelection {
3693                        provider: default_provider.clone(),
3694                        model: default_model.clone(),
3695                    },
3696                    transcript: &transcript,
3697                    tools: &routing_tools,
3698                    candidates: routing_candidates.as_deref(),
3699                    prior_failures,
3700                    prior_escalations: routing_escalations,
3701                },
3702            )
3703            .await;
3704            if let Some(decision) = routing_selection.decision.clone() {
3705                if matches!(decision.outcome, InferenceRoutingOutcome::Escalated) {
3706                    routing_escalations = routing_escalations.saturating_add(1);
3707                }
3708                self.emit(RoderEvent::InferenceRoutingDecision(
3709                    InferenceRoutingDecisionEvent {
3710                        thread_id: req.thread_id.clone(),
3711                        turn_id: turn_id.clone(),
3712                        round_index: round_index as u32,
3713                        default_selection: ModelSelection {
3714                            provider: default_provider.clone(),
3715                            model: default_model.clone(),
3716                        },
3717                        selected_selection: routing_selection.selection.clone(),
3718                        decision,
3719                        timestamp: OffsetDateTime::now_utc(),
3720                    },
3721                ))
3722                .await;
3723            }
3724            provider = routing_selection.selection.provider.clone();
3725            model = routing_selection.selection.model.clone();
3726            let engine = self.engine_for(&provider)?;
3727            let capabilities = engine.capabilities();
3728            let tool_images = ToolImages::for_engine(engine.as_ref(), &model);
3729            model_profile = model_profile_for_provider_model(&cfg, &provider, &model);
3730            let tools = if capabilities.tool_calls {
3731                if model == routing_tools_model && provider == routing_tools_provider {
3732                    routing_tools.clone()
3733                } else {
3734                    self.filtered_tool_specs(
3735                        &cfg,
3736                        &model,
3737                        &provider,
3738                        model_profile.as_ref(),
3739                        &thread_overrides.tool_allowlist,
3740                        &thread_overrides.external_tools,
3741                    )
3742                }
3743            } else {
3744                Vec::new()
3745            };
3746            let parallel_tool_calls = parallel_tool_calls_for_model(&cfg, &model);
3747            let tool_choice = if tools.is_empty() {
3748                ToolChoice::None
3749            } else {
3750                ToolChoice::Auto
3751            };
3752            let summary_selection = ModelSelection {
3753                provider: provider.clone(),
3754                model: model.clone(),
3755            };
3756            if model_switch_summary_selection.as_ref() != Some(&summary_selection) {
3757                if let Some(summary) = model_switch_summary(
3758                    &transcript,
3759                    model_profile.as_ref(),
3760                    &provider,
3761                    &model,
3762                    &tools,
3763                ) {
3764                    let item = TranscriptItem::UserMessage(UserMessage::text(summary));
3765                    self.persist_turn_item(&req.thread_id, &turn_id, &item)
3766                        .await?;
3767                    transcript.push(item);
3768                }
3769                model_switch_summary_selection = Some(summary_selection);
3770            }
3771
3772            if !capabilities.image_input && transcript_has_images(&transcript) {
3773                self.fail_turn_with_error(
3774                    &req.thread_id,
3775                    &turn_id,
3776                    format!("provider {provider} does not support image input"),
3777                )
3778                .await?;
3779                return Ok(TurnRunOutcome::Stopped);
3780            }
3781            let compaction_generation_before = self.compaction_generation(&req.thread_id);
3782            transcript = self
3783                .compact_transcript_if_needed(
3784                    &req.thread_id,
3785                    &turn_id,
3786                    &provider,
3787                    &model,
3788                    transcript,
3789                    self.compaction_options_for_turn(&req.thread_id, !compacted_this_turn),
3790                )
3791                .await?;
3792            compacted_this_turn |=
3793                self.compaction_generation(&req.thread_id) != compaction_generation_before;
3794            if round_index == 0 {
3795                self.complete_context_assembly(&req, &turn_id, &transcript, tool_images)
3796                    .await;
3797            }
3798
3799            let speed_policy_decision =
3800                speed_policy.decision(runtime_profile, &model, &cfg.speed_policy);
3801            let request_reasoning = reasoning_from_decision(
3802                speed_policy_decision.as_ref(),
3803                routing_selection
3804                    .reasoning
3805                    .clone()
3806                    .unwrap_or_else(|| reasoning_for_model(&cfg, &model)),
3807            );
3808            self.active_turn_selections.write().await.insert(
3809                turn_id.clone(),
3810                ModelSelectionMode::manual(
3811                    provider.clone(),
3812                    model.clone(),
3813                    request_reasoning.level.clone(),
3814                ),
3815            );
3816            if let Some(limit) = reliability.record_model_call(
3817                &cfg.reliability,
3818                runtime_profile == RuntimeProfile::Interactive,
3819            ) {
3820                self.fail_turn_due_to_reliability_limit(
3821                    &req.thread_id,
3822                    &turn_id,
3823                    &provider,
3824                    &model,
3825                    limit,
3826                    &transcript,
3827                )
3828                .await?;
3829                return Ok(TurnRunOutcome::Stopped);
3830            }
3831            self.emit(RoderEvent::InferenceStarted(InferenceStarted {
3832                thread_id: req.thread_id.clone(),
3833                turn_id: turn_id.clone(),
3834                engine_id: engine.id(),
3835                model: ModelSelection {
3836                    provider: provider.clone(),
3837                    model: model.clone(),
3838                },
3839                reasoning: request_reasoning.clone(),
3840                speed_policy: speed_policy_decision.clone(),
3841                deadline_remaining_seconds: deadline_remaining_seconds(turn_deadline),
3842                timestamp: OffsetDateTime::now_utc(),
3843            }))
3844            .await;
3845
3846            let mut instructions = req.instructions.clone();
3847            if let Some(extra) = &thread_overrides.developer_instructions {
3848                instructions = apply_thread_developer_instructions(instructions, extra);
3849            }
3850            if let Some(context) = req.developer_context.as_deref() {
3851                instructions = apply_turn_developer_context(instructions, context);
3852            }
3853            let mut instructions = apply_runtime_profile(instructions, runtime_profile);
3854            if let Some(profile) = &model_profile {
3855                instructions = apply_model_instruction_overlay(instructions, profile);
3856            }
3857            if req.task_ledger_required
3858                && runtime_profile == RuntimeProfile::Eval
3859                && !transcript_has_task_ledger(&transcript)
3860            {
3861                instructions = apply_task_ledger_required(instructions);
3862            }
3863            let effective_policy_mode = self.effective_policy_mode_for_thread(&req.thread_id).await;
3864            if effective_policy_mode == PolicyMode::Plan {
3865                instructions = apply_plan_mode(instructions);
3866            }
3867            if agent_swarm_mode_active {
3868                instructions = apply_agent_swarm_mode(instructions);
3869            }
3870            // Ultra mode (any model) enables proactive multi-agent policy.
3871            // Codex Sol/Terra Ultra effort still does too without requiring the
3872            // mode flag. Models that advertise Ultra effort keep the
3873            // explicit-request-only policy on lower efforts when ultra mode is
3874            // off; other models get multi-agent policy only when ultra mode is on.
3875            let model_has_ultra_effort = model_supports_reasoning_effort(&model, REASONING_ULTRA);
3876            let effort_is_ultra = request_reasoning.level.as_deref() == Some(REASONING_ULTRA);
3877            let proactive_multi_agent = ultra_mode_active || effort_is_ultra;
3878            if ultra_mode_active || model_has_ultra_effort {
3879                instructions = apply_codex_multi_agent_mode(instructions, proactive_multi_agent);
3880            }
3881            let compaction_instructions = instructions.clone();
3882            instructions = self
3883                .goals
3884                .apply_goal_instructions(&req.thread_id, instructions, effective_policy_mode)
3885                .await?;
3886            let mut request_metadata = serde_json::json!({});
3887            if let Some(decision) = &speed_policy_decision {
3888                request_metadata["speedPolicy"] = serde_json::json!(decision);
3889            }
3890            if let Some(decision) = routing_selection.decision.as_ref() {
3891                request_metadata["inferenceRouting"] = serde_json::json!(decision);
3892            }
3893            if let Some(remaining) = deadline_remaining_seconds(turn_deadline) {
3894                request_metadata["deadlineRemainingSeconds"] = serde_json::json!(remaining);
3895            }
3896            if let Some(profile) = &model_profile {
3897                request_metadata["modelProfile"] = serde_json::json!({
3898                    "model": profile.model,
3899                    "providerFamily": profile.provider_family,
3900                    "editTool": profile.edit_tool,
3901                    "schemaPolicy": profile.schema_policy,
3902                    "instructionOverlay": profile.instruction_overlay,
3903                    "parallelToolCalls": profile.parallel_tool_calls,
3904                    "autoCompactTokenLimit": profile.auto_compact_token_limit,
3905                });
3906            }
3907            let task_ledger_required_this_round = req.task_ledger_required
3908                && runtime_profile == RuntimeProfile::Eval
3909                && !deadline_finalization_requested
3910                && !transcript_has_task_ledger(&transcript);
3911            let task_ledger_tools = (capabilities.tool_calls && task_ledger_required_this_round)
3912                .then(|| self.task_ledger_tool_specs(model_profile.as_ref()))
3913                .filter(|tools| !tools.is_empty());
3914            let request_tools = if deadline_finalization_requested {
3915                Vec::new()
3916            } else if let Some(ledger_tools) = &task_ledger_tools {
3917                ledger_tools.clone()
3918            } else {
3919                tools.clone()
3920            };
3921            // Tool guidance describes the tools this request can call, which is
3922            // fewer than `tools` on a deadline finalization or forced ledger round.
3923            instructions = apply_parallel_web_tools(
3924                instructions,
3925                request_tools.iter().map(|spec| spec.name.as_str()),
3926            );
3927            instructions = apply_browser_routing(
3928                instructions,
3929                request_tools.iter().map(|spec| spec.name.as_str()),
3930            );
3931            let request_tool_choice = if deadline_finalization_requested {
3932                ToolChoice::None
3933            } else if task_ledger_tools.is_some() {
3934                ToolChoice::Specific(TASK_LEDGER_TOOL_NAME.to_string())
3935            } else {
3936                tool_choice.clone()
3937            };
3938            if deadline_finalization_requested {
3939                request_metadata["deadlineFinalization"] = serde_json::json!({
3940                    "reserveSeconds": deadline_finalization_reserve,
3941                    "remainingSeconds": deadline_remaining_seconds(turn_deadline),
3942                });
3943            } else if deadline_scoreable_completion_requested {
3944                request_metadata["deadlineScoreableCompletion"] = serde_json::json!({
3945                    "reserveSeconds": deadline_finalization_reserve,
3946                    "remainingSeconds": deadline_remaining_seconds(turn_deadline),
3947                });
3948            }
3949            let mut eager_tools = EagerTools::new(
3950                self,
3951                &request_tools,
3952                &thread_overrides.external_tools,
3953                parallel_tool_calls,
3954            );
3955            let mut request_reliability: ReliabilityRequestPolicy = cfg.reliability.clone().into();
3956            request_reliability.provider_retry_max_attempts = request_reliability
3957                .provider_retry_max_attempts
3958                .saturating_sub(provider_stream_retry_attempts)
3959                .max(1);
3960            let request = AgentInferenceRequest {
3961                model: ModelSelection {
3962                    provider: provider.clone(),
3963                    model: model.clone(),
3964                },
3965                instructions,
3966                transcript: crate::tool_result_images::request_transcript(&transcript, tool_images),
3967                tools: request_tools,
3968                tool_choice: request_tool_choice,
3969                reasoning: request_reasoning,
3970                output: OutputConfig::default(),
3971                runtime: RuntimeHints {
3972                    auto_compact_token_limit: (provider != roder_api::catalog::PROVIDER_CODEX)
3973                        .then(|| server_side_compaction_threshold(&cfg, &model))
3974                        .flatten(),
3975                    profile: runtime_profile,
3976                    parallel_tool_calls: Some(parallel_tool_calls),
3977                    prompt_cache_key: Some(req.thread_id.clone()),
3978                    hosted_web_search: cfg.hosted_web_search.clone(),
3979                    tool_search: tool_search_for_provider_model(&cfg, &provider, &model),
3980                    speed_policy: speed_policy_decision,
3981                    reliability: Some(request_reliability),
3982                    deadline_remaining_seconds: deadline_remaining_seconds(turn_deadline),
3983                    service_tier: req.service_tier_override.clone(),
3984                    ..RuntimeHints::default()
3985                },
3986                metadata: request_metadata,
3987            };
3988
3989            let mut compaction_template = request.clone();
3990            compaction_template.instructions = compaction_instructions;
3991            compaction_template.transcript.clear();
3992            self.compaction_templates
3993                .write()
3994                .await
3995                .insert(req.thread_id.clone(), compaction_template);
3996
3997            let ctx = InferenceTurnContext {
3998                thread_id: &req.thread_id,
3999                turn_id: &turn_id,
4000                tool_executor: Some(std::sync::Arc::new(
4001                    crate::tool_execution::RuntimeTurnToolExecutor {
4002                        runtime: Arc::clone(self),
4003                        thread_id: req.thread_id.clone(),
4004                        turn_id: turn_id.clone(),
4005                        workspace: Some(workspace.clone()),
4006                        deadline: turn_deadline,
4007                    },
4008                )),
4009            };
4010            let stream_future = preemptible(engine.stream_turn(ctx, request), &mut steering);
4011            let mut stream = if let Some((deadline, timeout_action)) = inference_timeout_deadline(
4012                turn_deadline,
4013                runtime_profile,
4014                req.task_ledger_required,
4015                deadline_finalization_reserve,
4016                deadline_finalization_requested || deadline_scoreable_completion_requested,
4017                task_ledger_scoreable_checkpoints,
4018                &transcript,
4019            ) {
4020                match tokio::time::timeout_at(deadline_instant(deadline), stream_future).await {
4021                    Ok(stream) => match stream {
4022                        Ok(stream) => stream,
4023                        Err(err) => {
4024                            if err.is::<SamplingPreempted>() {
4025                                self.finish_sampling_cleanup(&turn_id).await?;
4026                                provider_stream_retry_attempts = 0;
4027                                continue 'tool_rounds;
4028                            }
4029                            self.finish_sampling_cleanup(&turn_id).await?;
4030                            if !deadline_finalization_requested
4031                                && self
4032                                    .retry_sampling_failure(
4033                                        SamplingRetry {
4034                                            thread_id: &req.thread_id,
4035                                            turn_id: &turn_id,
4036                                            provider: &provider,
4037                                            model: &model,
4038                                            config: &cfg.reliability,
4039                                            error: &err,
4040                                        },
4041                                        &mut provider_stream_retry_attempts,
4042                                        &mut steering,
4043                                    )
4044                                    .await
4045                            {
4046                                continue 'tool_rounds;
4047                            }
4048                            let error = err.to_string();
4049                            if self
4050                                .try_recover_from_context_limit(
4051                                    &req.thread_id,
4052                                    &turn_id,
4053                                    &provider,
4054                                    &model,
4055                                    &error,
4056                                    &mut transcript,
4057                                    &mut compacted_this_turn,
4058                                    &mut context_limit_recovery_attempts,
4059                                )
4060                                .await?
4061                            {
4062                                continue 'tool_rounds;
4063                            }
4064                            return Err(err);
4065                        }
4066                    },
4067                    Err(_) => {
4068                        if runtime_profile == RuntimeProfile::Eval
4069                            && !deadline_finalization_requested
4070                        {
4071                            let remaining = deadline_remaining_seconds(turn_deadline).unwrap_or(0);
4072                            if timeout_action == InferenceTimeoutAction::ScoreableCheckpoint
4073                                && task_ledger_scoreable_checkpoints
4074                                    < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
4075                                && let Some(prompt) = task_ledger_completion_prompt(&transcript)
4076                            {
4077                                task_ledger_scoreable_checkpoints += 1;
4078                                let item = TranscriptItem::UserMessage(UserMessage::text(
4079                                    task_ledger_scoreable_checkpoint_prompt(remaining, &prompt),
4080                                ));
4081                                self.persist_turn_item(&req.thread_id, &turn_id, &item)
4082                                    .await?;
4083                                transcript.push(item);
4084                                continue 'tool_rounds;
4085                            }
4086                            self.start_deadline_finalization(
4087                                &req.thread_id,
4088                                &turn_id,
4089                                &mut transcript,
4090                                remaining,
4091                            )
4092                            .await?;
4093                            deadline_finalization_requested = true;
4094                            continue 'tool_rounds;
4095                        }
4096                        self.fail_turn_due_to_deadline(
4097                            &req.thread_id,
4098                            &turn_id,
4099                            deadline,
4100                            &transcript,
4101                        )
4102                        .await?;
4103                        return Ok(TurnRunOutcome::Stopped);
4104                    }
4105                }
4106            } else {
4107                match stream_future.await {
4108                    Ok(stream) => stream,
4109                    Err(err) => {
4110                        if err.is::<SamplingPreempted>() {
4111                            self.finish_sampling_cleanup(&turn_id).await?;
4112                            provider_stream_retry_attempts = 0;
4113                            continue 'tool_rounds;
4114                        }
4115                        self.finish_sampling_cleanup(&turn_id).await?;
4116                        if !deadline_finalization_requested
4117                            && self
4118                                .retry_sampling_failure(
4119                                    SamplingRetry {
4120                                        thread_id: &req.thread_id,
4121                                        turn_id: &turn_id,
4122                                        provider: &provider,
4123                                        model: &model,
4124                                        config: &cfg.reliability,
4125                                        error: &err,
4126                                    },
4127                                    &mut provider_stream_retry_attempts,
4128                                    &mut steering,
4129                                )
4130                                .await
4131                        {
4132                            continue 'tool_rounds;
4133                        }
4134                        let error = err.to_string();
4135                        if self
4136                            .try_recover_from_context_limit(
4137                                &req.thread_id,
4138                                &turn_id,
4139                                &provider,
4140                                &model,
4141                                &error,
4142                                &mut transcript,
4143                                &mut compacted_this_turn,
4144                                &mut context_limit_recovery_attempts,
4145                            )
4146                            .await?
4147                        {
4148                            continue 'tool_rounds;
4149                        }
4150                        return Err(err);
4151                    }
4152                }
4153            };
4154            let mut sampling_output = SamplingOutput::default();
4155            let output_context = OutputContext {
4156                thread_id: &req.thread_id,
4157                turn_id: &turn_id,
4158                profile: model_profile.as_ref(),
4159                provider: &provider,
4160                model: &model,
4161            };
4162            let mut assistant_text = String::new();
4163            let mut phase_messages = Vec::<AssistantMessage>::new();
4164            let mut reasoning_text = String::new();
4165            let mut replayed_tool_call = false;
4166            let mut tool_calls = Vec::new();
4167            let mut patch_progress = PatchProgressTracker::default();
4168            let mut provider_metadata = None;
4169            let mut provider_requests_continuation = false;
4170            // Captured from mid-stream Compaction(completed) events. Codex/OpenAI
4171            // sometimes abort the SSE stream after emitting the compaction item
4172            // ("error decoding response body") before response.completed, so we
4173            // must not rely solely on ProviderMetadata for the boundary.
4174            let mut stream_compaction_item: Option<serde_json::Value> = None;
4175
4176            loop {
4177                let next_future = async {
4178                    loop {
4179                        tokio::select! {
4180                            biased;
4181                            _ = wait_for_steer(&mut steering) => break Some(Err(SamplingPreempted.into())),
4182                            result = eager_tools.poll_result(), if eager_tools.pending() => {
4183                                if let Err(error) = result { break Some(Err(error)); }
4184                            }
4185                            next = stream.next() => break next,
4186                        }
4187                    }
4188                };
4189                let next = if let Some((deadline, timeout_action)) = inference_timeout_deadline(
4190                    turn_deadline,
4191                    runtime_profile,
4192                    req.task_ledger_required,
4193                    deadline_finalization_reserve,
4194                    deadline_finalization_requested || deadline_scoreable_completion_requested,
4195                    task_ledger_scoreable_checkpoints,
4196                    &transcript,
4197                ) {
4198                    match tokio::time::timeout_at(deadline_instant(deadline), next_future).await {
4199                        Ok(next) => next,
4200                        Err(_) => {
4201                            if runtime_profile == RuntimeProfile::Eval
4202                                && !deadline_finalization_requested
4203                            {
4204                                let remaining =
4205                                    deadline_remaining_seconds(turn_deadline).unwrap_or(0);
4206                                if timeout_action == InferenceTimeoutAction::ScoreableCheckpoint
4207                                    && task_ledger_scoreable_checkpoints
4208                                        < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
4209                                    && let Some(prompt) = task_ledger_completion_prompt(&transcript)
4210                                {
4211                                    task_ledger_scoreable_checkpoints += 1;
4212                                    let item = TranscriptItem::UserMessage(UserMessage::text(
4213                                        task_ledger_scoreable_checkpoint_prompt(remaining, &prompt),
4214                                    ));
4215                                    self.persist_turn_item(&req.thread_id, &turn_id, &item)
4216                                        .await?;
4217                                    transcript.push(item);
4218                                    continue 'tool_rounds;
4219                                }
4220                                self.start_deadline_finalization(
4221                                    &req.thread_id,
4222                                    &turn_id,
4223                                    &mut transcript,
4224                                    remaining,
4225                                )
4226                                .await?;
4227                                deadline_finalization_requested = true;
4228                                continue 'tool_rounds;
4229                            }
4230                            self.fail_turn_due_to_deadline(
4231                                &req.thread_id,
4232                                &turn_id,
4233                                deadline,
4234                                &transcript,
4235                            )
4236                            .await?;
4237                            return Ok(TurnRunOutcome::Stopped);
4238                        }
4239                    }
4240                } else {
4241                    next_future.await
4242                };
4243                let res = next.unwrap_or_else(|| {
4244                    Err(roder_api::provider_error::ProviderFailure::new(
4245                        roder_api::provider_error::ProviderFailureKind::StreamInterrupted,
4246                        "provider stream ended before terminal completion",
4247                    )
4248                    .into())
4249                });
4250                let event = match res {
4251                    Ok(event) => event,
4252                    Err(err) => {
4253                        drop(stream);
4254                        eager_tools.drain().await?;
4255                        for result in eager_tools.take_results(&tool_calls) {
4256                            verification_gate.record_tool_result(&result);
4257                            transcript.push(TranscriptItem::ToolResult(result));
4258                        }
4259                        self.finish_sampling_cleanup(&turn_id).await?;
4260                        if err.is::<SamplingPreempted>() {
4261                            provider_stream_retry_attempts = 0;
4262                            continue 'tool_rounds;
4263                        }
4264                        if !deadline_finalization_requested
4265                            && self
4266                                .retry_sampling_failure(
4267                                    SamplingRetry {
4268                                        thread_id: &req.thread_id,
4269                                        turn_id: &turn_id,
4270                                        provider: &provider,
4271                                        model: &model,
4272                                        config: &cfg.reliability,
4273                                        error: &err,
4274                                    },
4275                                    &mut provider_stream_retry_attempts,
4276                                    &mut steering,
4277                                )
4278                                .await
4279                        {
4280                            continue 'tool_rounds;
4281                        }
4282                        let error = err.to_string();
4283                        if self
4284                            .try_recover_from_context_limit(
4285                                &req.thread_id,
4286                                &turn_id,
4287                                &provider,
4288                                &model,
4289                                &error,
4290                                &mut transcript,
4291                                &mut compacted_this_turn,
4292                                &mut context_limit_recovery_attempts,
4293                            )
4294                            .await?
4295                        {
4296                            continue 'tool_rounds;
4297                        }
4298                        return Err(err);
4299                    }
4300                };
4301
4302                let inference_timestamp = OffsetDateTime::now_utc();
4303                self.emit(RoderEvent::InferenceEventReceived(InferenceEventReceived {
4304                    thread_id: req.thread_id.clone(),
4305                    turn_id: turn_id.clone(),
4306                    event: event.clone(),
4307                    timestamp: inference_timestamp,
4308                }))
4309                .await;
4310
4311                match event {
4312                    InferenceEvent::MessageDelta(delta) => {
4313                        if !delta.text.trim().is_empty() {
4314                            self.goals.record_activity(&turn_id).await;
4315                        }
4316                        if let Some((team_id, member)) =
4317                            self.teams.member_for_thread(&req.thread_id).await
4318                        {
4319                            self.emit(RoderEvent::TeamMemberMessageDelta(TeamMemberMessageDelta {
4320                                team_id,
4321                                member_id: member.id,
4322                                member_thread_id: req.thread_id.clone(),
4323                                turn_id: turn_id.clone(),
4324                                delta: delta.text.clone(),
4325                                timestamp: OffsetDateTime::now_utc(),
4326                            }))
4327                            .await;
4328                        }
4329                        if is_final_answer_phase(delta.phase.as_deref()) {
4330                            assistant_text.push_str(&delta.text);
4331                        } else if let Some(last) = phase_messages.last_mut()
4332                            && last.phase == delta.phase
4333                        {
4334                            last.text.push_str(&delta.text);
4335                        } else {
4336                            phase_messages.push(AssistantMessage {
4337                                text: delta.text,
4338                                phase: delta.phase,
4339                            });
4340                        }
4341                    }
4342                    InferenceEvent::ReasoningDelta(delta) => reasoning_text.push_str(&delta.text),
4343                    InferenceEvent::ToolCallCompleted(call) => {
4344                        self.goals.record_activity(&turn_id).await;
4345                        if completed_call_replayed(&transcript, &call)? {
4346                            replayed_tool_call = true;
4347                            continue;
4348                        }
4349                        if let Some(progress) = patch_progress.complete(
4350                            &req.thread_id,
4351                            &turn_id,
4352                            &call.id,
4353                            &call.name,
4354                            &call.arguments,
4355                        ) {
4356                            self.emit(RoderEvent::PatchProgress(progress)).await;
4357                        }
4358                        if !tool_calls
4359                            .iter()
4360                            .any(|previous: &ToolCallCompleted| previous.id == call.id)
4361                        {
4362                            eager_tools
4363                                .schedule(
4364                                    self,
4365                                    &output_context,
4366                                    &call,
4367                                    workspace.as_str(),
4368                                    turn_deadline,
4369                                    &mut transcript,
4370                                )
4371                                .await?;
4372                            tool_calls.push(call);
4373                        }
4374                    }
4375                    InferenceEvent::OutputItemCompleted(item) => {
4376                        sampling_output
4377                            .complete_item(self, &output_context, item, &mut transcript)
4378                            .await?;
4379                    }
4380                    InferenceEvent::Failed(failure) => {
4381                        speed_policy.record_failure();
4382                        let error = failure.message;
4383                        if self
4384                            .try_recover_from_context_limit(
4385                                &req.thread_id,
4386                                &turn_id,
4387                                &provider,
4388                                &model,
4389                                &error,
4390                                &mut transcript,
4391                                &mut compacted_this_turn,
4392                                &mut context_limit_recovery_attempts,
4393                            )
4394                            .await?
4395                        {
4396                            continue 'tool_rounds;
4397                        }
4398                        self.persist_turn_item(
4399                            &req.thread_id,
4400                            &turn_id,
4401                            &TranscriptItem::Error(ErrorRecord {
4402                                message: error.clone(),
4403                            }),
4404                        )
4405                        .await?;
4406                        self.emit(RoderEvent::TurnFailed(TurnFailed {
4407                            thread_id: req.thread_id.clone(),
4408                            turn_id: turn_id.clone(),
4409                            error: error.clone(),
4410                            error_kind: None,
4411                            usage: None,
4412                            timestamp: OffsetDateTime::now_utc(),
4413                        }))
4414                        .await;
4415                        self.complete_team_member_turn_with_result(
4416                            &req.thread_id,
4417                            &turn_id,
4418                            TeamMemberStatus::Failed,
4419                            (!assistant_text.trim().is_empty()).then(|| assistant_text.clone()),
4420                            Some(error),
4421                        )
4422                        .await?;
4423                        return Ok(TurnRunOutcome::Stopped);
4424                    }
4425                    InferenceEvent::Usage(usage) => {
4426                        self.record_goal_token_usage(
4427                            &req.thread_id,
4428                            &turn_id,
4429                            usage.total_tokens as i64,
4430                        )
4431                        .await?;
4432                        turn_usage.add_assign(&usage);
4433                    }
4434                    InferenceEvent::Completed(metadata) => {
4435                        sampling_output
4436                            .finish(
4437                                self,
4438                                &output_context,
4439                                &assistant_text,
4440                                &phase_messages,
4441                                &reasoning_text,
4442                                &mut transcript,
4443                            )
4444                            .await?;
4445                        provider_stream_retry_attempts = 0;
4446                        turn_finish_reason = metadata
4447                            .stop_reason
4448                            .as_deref()
4449                            .map(finish_reason_from_stop_reason);
4450                        break;
4451                    }
4452                    InferenceEvent::Compaction(progress) => {
4453                        // Persist the boundary as soon as the provider emits a
4454                        // completed compaction item. ChatGPT Codex often aborts
4455                        // the SSE stream right after ("error decoding response
4456                        // body"), so waiting for ProviderMetadata loses the
4457                        // boundary and every subsequent request re-compacts.
4458                        if progress.status == "completed"
4459                            && let Some(item) = progress.item
4460                        {
4461                            let already = stream_compaction_item
4462                                .as_ref()
4463                                .and_then(|existing| {
4464                                    existing.get("id").and_then(serde_json::Value::as_str)
4465                                })
4466                                .zip(item.get("id").and_then(serde_json::Value::as_str))
4467                                .is_some_and(|(a, b)| a == b);
4468                            if !already {
4469                                stream_compaction_item = Some(item.clone());
4470                                let boundary = TranscriptItem::ProviderMetadata(
4471                                    crate::compaction::provider_metadata_from_compaction_item(item),
4472                                );
4473                                self.persist_turn_item(&req.thread_id, &turn_id, &boundary)
4474                                    .await?;
4475                                transcript.push(boundary);
4476                                transcript =
4477                                    crate::compaction::trim_to_last_compaction_boundary(transcript);
4478                                compacted_this_turn = true;
4479                            }
4480                        }
4481                    }
4482                    InferenceEvent::HostedToolCallStarted(_)
4483                    | InferenceEvent::HostedToolCallCompleted(_) => {}
4484                    InferenceEvent::ToolCallStarted(call) => {
4485                        patch_progress.start(&call.id, &call.name)
4486                    }
4487                    InferenceEvent::ToolCallDelta(delta) => {
4488                        if let Some(progress) = patch_progress.delta(
4489                            &req.thread_id,
4490                            &turn_id,
4491                            &delta.id,
4492                            &delta.arguments_delta,
4493                        ) {
4494                            self.emit(RoderEvent::PatchProgress(progress)).await;
4495                        }
4496                    }
4497                    InferenceEvent::ProviderMetadata(mut metadata) => {
4498                        if metadata.get("kind").and_then(serde_json::Value::as_str)
4499                            == Some("reliability_retry_attempt")
4500                        {
4501                            provider_stream_retry_attempts =
4502                                provider_stream_retry_attempts.saturating_add(1);
4503                        }
4504                        provider_requests_continuation |= metadata
4505                            .get("end_turn")
4506                            .and_then(serde_json::Value::as_bool)
4507                            == Some(false);
4508                        if let Some(compaction_item) = stream_compaction_item.as_ref() {
4509                            let _ = crate::compaction::ensure_provider_metadata_compaction(
4510                                &mut metadata,
4511                                compaction_item,
4512                            );
4513                        }
4514                        if metadata.get("output").is_none() {
4515                            let item = TranscriptItem::ProviderMetadata(metadata);
4516                            self.persist_turn_item(&req.thread_id, &turn_id, &item)
4517                                .await?;
4518                            transcript.push(item);
4519                        } else {
4520                            provider_metadata = Some(metadata);
4521                        }
4522                    }
4523                }
4524            }
4525
4526            speed_policy.record_model_output(
4527                !assistant_text.is_empty() || !phase_messages.is_empty(),
4528                tool_calls.len(),
4529            );
4530            if tool_calls.is_empty() {
4531                if provider_requests_continuation || replayed_tool_call {
4532                    if let Some(metadata) = provider_metadata {
4533                        let item = TranscriptItem::ProviderMetadata(metadata);
4534                        self.persist_turn_item(&req.thread_id, &turn_id, &item)
4535                            .await?;
4536                        transcript.push(item);
4537                    }
4538                    continue 'tool_rounds;
4539                }
4540                let steers = self.drain_turn_steers(&turn_id, &mut steering).await;
4541                if !steers.is_empty() {
4542                    if let Some(metadata) = provider_metadata {
4543                        let had_provider_compaction =
4544                            crate::compaction::provider_metadata_has_compaction(&metadata);
4545                        let item = TranscriptItem::ProviderMetadata(metadata);
4546                        self.persist_turn_item(&req.thread_id, &turn_id, &item)
4547                            .await?;
4548                        transcript.push(item);
4549                        if had_provider_compaction {
4550                            // Server-side compaction replaced the prior window;
4551                            // drop pre-boundary items so the next request does
4552                            // not re-send (and re-compact) the full history.
4553                            transcript =
4554                                crate::compaction::trim_to_last_compaction_boundary(transcript);
4555                            compacted_this_turn = true;
4556                        }
4557                    }
4558                    self.append_steers(&req, &turn_id, &mut transcript, steers)
4559                        .await?;
4560                    continue;
4561                }
4562                if !deadline_finalization_requested
4563                    && req.task_ledger_required
4564                    && runtime_profile == RuntimeProfile::Eval
4565                    && task_ledger_completion_reminders < TASK_LEDGER_COMPLETION_REMINDER_LIMIT
4566                    && (!assistant_text.trim().is_empty() || !phase_messages.is_empty())
4567                    && let Some(prompt) = task_ledger_completion_prompt(&transcript)
4568                {
4569                    task_ledger_completion_reminders += 1;
4570                    let item = TranscriptItem::UserMessage(UserMessage::text(prompt));
4571                    self.persist_turn_item(&req.thread_id, &turn_id, &item)
4572                        .await?;
4573                    transcript.push(item);
4574                    continue;
4575                }
4576                if !deadline_finalization_requested
4577                    && let Some(prompt) = verification_gate.blocking_prompt()
4578                {
4579                    speed_policy.record_verification_required();
4580                    self.emit(RoderEvent::VerificationRequired(VerificationRequired {
4581                        thread_id: req.thread_id.clone(),
4582                        turn_id: turn_id.clone(),
4583                        reason: verification_gate.reason(),
4584                        changed_files: verification_gate.changed_files(),
4585                        tool_evidence: verification_gate.tool_evidence.clone(),
4586                        tests_run: verification_gate.tests_run.clone(),
4587                        open_gaps: verification_gate.open_gaps.clone(),
4588                        timestamp: OffsetDateTime::now_utc(),
4589                    }))
4590                    .await;
4591                    let item = TranscriptItem::UserMessage(UserMessage::text(prompt));
4592                    self.persist_turn_item(&req.thread_id, &turn_id, &item)
4593                        .await?;
4594                    transcript.push(item);
4595                    continue;
4596                }
4597                // Loop persistence nudge: in non-interactive/eval runs, when the
4598                // model returns a final message with no tool calls, gently prompt
4599                // it to keep going (bounded by `empty_tool_call_nudges`) before
4600                // ending the turn. Gated to non-interactive/eval so interactive
4601                // chat completions are never delayed, and only fires when the
4602                // model actually produced a final answer to re-examine.
4603                if !deadline_finalization_requested
4604                    && runtime_profile != RuntimeProfile::Interactive
4605                    && cfg.reliability.empty_tool_call_nudges > 0
4606                    && empty_tool_call_nudges_used < cfg.reliability.empty_tool_call_nudges
4607                    && (!assistant_text.trim().is_empty() || !phase_messages.is_empty())
4608                {
4609                    empty_tool_call_nudges_used += 1;
4610                    let item = TranscriptItem::UserMessage(UserMessage::text(
4611                        RELIABILITY_CONTINUATION_PROMPT.to_string(),
4612                    ));
4613                    self.persist_turn_item(&req.thread_id, &turn_id, &item)
4614                        .await?;
4615                    transcript.push(item);
4616                    continue;
4617                }
4618                if deadline_finalization_requested
4619                    && assistant_text.trim().is_empty()
4620                    && phase_messages.is_empty()
4621                {
4622                    assistant_text = format!(
4623                        "Deadline finalization completed without model text. {}",
4624                        turn_partial_result(&transcript)
4625                    );
4626                }
4627                final_assistant_text = assistant_text;
4628                final_provider_metadata = provider_metadata;
4629                exhausted_tool_rounds = false;
4630                break;
4631            }
4632
4633            if let Some(metadata) = provider_metadata {
4634                let had_provider_compaction =
4635                    crate::compaction::provider_metadata_has_compaction(&metadata);
4636                let item = TranscriptItem::ProviderMetadata(metadata);
4637                self.persist_turn_item(&req.thread_id, &turn_id, &item)
4638                    .await?;
4639                transcript.push(item);
4640                if had_provider_compaction {
4641                    // Server-side compaction replaced the prior window; drop
4642                    // pre-boundary items so subsequent tool rounds use the
4643                    // compact window as the next input.
4644                    transcript = crate::compaction::trim_to_last_compaction_boundary(transcript);
4645                    compacted_this_turn = true;
4646                }
4647            }
4648            eager_tools.drain().await?;
4649            for call in &tool_calls {
4650                if eager_tools.scheduled(&call.id) {
4651                    continue;
4652                }
4653                let tool_item = TranscriptItem::ToolCall(ToolCallRecord {
4654                    id: call.id.clone(),
4655                    name: call.name.clone(),
4656                    arguments: call.arguments.clone(),
4657                });
4658                self.persist_turn_item(&req.thread_id, &turn_id, &tool_item)
4659                    .await?;
4660                transcript.push(tool_item);
4661                self.persist_model_profile_segment(
4662                    &req.thread_id,
4663                    &turn_id,
4664                    model_profile.as_ref(),
4665                    &provider,
4666                    &model,
4667                    "tool_call",
4668                )
4669                .await?;
4670            }
4671            if let Some(deadline) = turn_deadline
4672                && deadline_expired(deadline)
4673            {
4674                self.fail_turn_due_to_deadline(&req.thread_id, &turn_id, deadline, &transcript)
4675                    .await?;
4676                return Ok(TurnRunOutcome::Stopped);
4677            }
4678            let mut results = eager_tools.take_results(&tool_calls);
4679            let remaining_calls = tool_calls
4680                .into_iter()
4681                .filter(|call| !eager_tools.scheduled(&call.id))
4682                .collect();
4683            results.extend(
4684                self.route_tool_calls(
4685                    &req.thread_id,
4686                    &turn_id,
4687                    remaining_calls,
4688                    parallel_tool_calls,
4689                    Some(workspace.as_str()),
4690                    turn_deadline,
4691                )
4692                .await?,
4693            );
4694            let reliability_limit = reliability.record_tool_results(
4695                &cfg.reliability,
4696                &results,
4697                runtime_profile == RuntimeProfile::Interactive,
4698            );
4699            for result in results {
4700                verification_gate.record_tool_result(&result);
4701                transcript.push(TranscriptItem::ToolResult(result));
4702                self.persist_model_profile_segment(
4703                    &req.thread_id,
4704                    &turn_id,
4705                    model_profile.as_ref(),
4706                    &provider,
4707                    &model,
4708                    "tool_result",
4709                )
4710                .await?;
4711            }
4712            if let Some(limit) = reliability_limit {
4713                if limit.decision == ReliabilityLimitDecision::RequestContinuation {
4714                    // Loop persistence: rather than ending the turn, reset the
4715                    // consecutive-failure counter, nudge the model to keep going,
4716                    // and continue the round loop. Bounded by the per-turn tool
4717                    // failure ceiling, `max_model_calls_per_turn`, and
4718                    // `MAX_TOOL_ROUNDS_PER_TURN`.
4719                    self.record_reliability_limit_continuation(
4720                        &req.thread_id,
4721                        &turn_id,
4722                        &provider,
4723                        &model,
4724                        limit,
4725                    )
4726                    .await;
4727                    reliability.reset_consecutive_failures();
4728                    let nudge = TranscriptItem::UserMessage(UserMessage::text(
4729                        RELIABILITY_CONTINUATION_PROMPT.to_string(),
4730                    ));
4731                    self.persist_turn_item(&req.thread_id, &turn_id, &nudge)
4732                        .await?;
4733                    transcript.push(nudge);
4734                } else {
4735                    self.fail_turn_due_to_reliability_limit(
4736                        &req.thread_id,
4737                        &turn_id,
4738                        &provider,
4739                        &model,
4740                        limit,
4741                        &transcript,
4742                    )
4743                    .await?;
4744                    return Ok(TurnRunOutcome::Stopped);
4745                }
4746            }
4747            let compaction_generation_before = self.compaction_generation(&req.thread_id);
4748            transcript = self
4749                .compact_transcript_if_needed(
4750                    &req.thread_id,
4751                    &turn_id,
4752                    &provider,
4753                    &model,
4754                    transcript,
4755                    self.compaction_options_for_turn(&req.thread_id, !compacted_this_turn),
4756                )
4757                .await?;
4758            compacted_this_turn |=
4759                self.compaction_generation(&req.thread_id) != compaction_generation_before;
4760        }
4761
4762        if exhausted_tool_rounds {
4763            let message =
4764                format!("tool call limit reached after {MAX_TOOL_ROUNDS_PER_TURN} rounds");
4765            self.persist_turn_item(
4766                &req.thread_id,
4767                &turn_id,
4768                &TranscriptItem::Error(ErrorRecord {
4769                    message: message.clone(),
4770                }),
4771            )
4772            .await?;
4773            self.emit(RoderEvent::TurnFailed(TurnFailed {
4774                thread_id: req.thread_id.clone(),
4775                turn_id: turn_id.clone(),
4776                error: message.clone(),
4777                error_kind: None,
4778                usage: None,
4779                timestamp: OffsetDateTime::now_utc(),
4780            }))
4781            .await;
4782            self.complete_team_member_turn_with_result(
4783                &req.thread_id,
4784                &turn_id,
4785                TeamMemberStatus::Failed,
4786                None,
4787                Some(message),
4788            )
4789            .await?;
4790            return Ok(TurnRunOutcome::Stopped);
4791        }
4792
4793        if let Some(metadata) = final_provider_metadata {
4794            self.persist_turn_item(
4795                &req.thread_id,
4796                &turn_id,
4797                &TranscriptItem::ProviderMetadata(metadata),
4798            )
4799            .await?;
4800        }
4801
4802        let completed_usage = (!turn_usage.is_empty()).then_some(turn_usage.clone());
4803        self.record_thread_usage_metadata(&req.thread_id, &turn_usage)
4804            .await?;
4805        let (cleanup, ownership) = self.await_provider_turn_cleanup(&turn_id).await;
4806        self.record_turn_lifecycle_with_ownership(
4807            req.thread_id.clone(),
4808            turn_id.clone(),
4809            TurnLifecycleState::Completed,
4810            cleanup,
4811            None,
4812            ownership,
4813        )
4814        .await;
4815        self.emit(RoderEvent::TurnCompleted(TurnCompleted {
4816            thread_id: req.thread_id.clone(),
4817            turn_id: turn_id.clone(),
4818            usage: completed_usage,
4819            finish_reason: turn_finish_reason,
4820            timestamp: OffsetDateTime::now_utc(),
4821        }))
4822        .await;
4823        self.complete_team_member_turn_with_result(
4824            &req.thread_id,
4825            &turn_id,
4826            TeamMemberStatus::Completed,
4827            (!final_assistant_text.is_empty()).then_some(final_assistant_text),
4828            None,
4829        )
4830        .await?;
4831        Ok(TurnRunOutcome::Completed)
4832    }
4833
4834    async fn route_tool_calls(
4835        self: &Arc<Self>,
4836        thread_id: &ThreadId,
4837        turn_id: &TurnId,
4838        calls: Vec<ToolCallCompleted>,
4839        parallel: bool,
4840        workspace: Option<&str>,
4841        deadline: Option<OffsetDateTime>,
4842    ) -> anyhow::Result<Vec<ToolResultRecord>> {
4843        // Enforce the agent_swarm exclusivity rule (roadmap 104, Task 2):
4844        // `agent_swarm` must be the only tool call in a model response. A mixed
4845        // or multi-swarm batch is denied wholesale with actionable retry text so
4846        // the model re-issues `agent_swarm` by itself, and every tool_call_id
4847        // still gets a response (keeping the chat-completions transcript valid).
4848        if let Some(violation) = roder_api::subagents::agent_swarm_batch_violation(
4849            calls.iter().map(|call| call.name.as_str()),
4850        ) {
4851            let message = violation.deny_message();
4852            return Ok(calls
4853                .into_iter()
4854                .map(|call| ToolResultRecord {
4855                    id: call.id,
4856                    name: Some(call.name),
4857                    result: message.clone(),
4858                    display_payload: None,
4859                    is_error: true,
4860                })
4861                .collect());
4862        }
4863        // Emit swarm lifecycle events on the bus (roadmap 104, Task 1) so any
4864        // app-server/SDK/TUI client can observe a swarm as a whole; per-child
4865        // progress flows through the existing Subagent* trace events.
4866        let swarm_call = (calls.len() == 1
4867            && calls[0].name == roder_api::subagents::AGENT_SWARM_TOOL_NAME)
4868            .then(|| (calls[0].id.clone(), calls[0].arguments.clone()));
4869        if let Some((tool_id, args)) = &swarm_call {
4870            self.emit(RoderEvent::AgentSwarmStarted(
4871                roder_api::subagents::AgentSwarmStarted {
4872                    thread_id: thread_id.clone(),
4873                    turn_id: turn_id.clone(),
4874                    tool_id: tool_id.clone(),
4875                    child_count: agent_swarm_child_count(args),
4876                    timestamp: OffsetDateTime::now_utc(),
4877                },
4878            ))
4879            .await;
4880        }
4881
4882        let force_sequential = calls
4883            .iter()
4884            .any(|call| crate::agent_control_tools::is_agent_control_tool(&call.name));
4885        let results =
4886            if parallel && !force_sequential {
4887                try_join_all(calls.into_iter().map(|call| {
4888                    self.route_tool_call(thread_id, turn_id, call, workspace, deadline)
4889                }))
4890                .await
4891            } else {
4892                let mut results = Vec::with_capacity(calls.len());
4893                for call in calls {
4894                    results.push(
4895                        self.route_tool_call(thread_id, turn_id, call, workspace, deadline)
4896                            .await?,
4897                    );
4898                }
4899                Ok(results)
4900            }?;
4901
4902        if let Some((tool_id, _)) = &swarm_call
4903            && let Some(result) = results.iter().find(|result| &result.id == tool_id)
4904            && let Some((completed, failed, aborted)) = parse_swarm_counts(&result.result)
4905        {
4906            self.emit(RoderEvent::AgentSwarmCompleted(
4907                roder_api::subagents::AgentSwarmCompleted {
4908                    thread_id: thread_id.clone(),
4909                    turn_id: turn_id.clone(),
4910                    tool_id: tool_id.clone(),
4911                    completed,
4912                    failed,
4913                    aborted,
4914                    timestamp: OffsetDateTime::now_utc(),
4915                },
4916            ))
4917            .await;
4918        }
4919
4920        Ok(results)
4921    }
4922
4923    /// On provider context/prompt-length failures, force-compact (and if needed
4924    /// strip the last bulky item) then retry the current tool round instead of
4925    /// failing the turn. Bounded so a permanently oversized suffix still stops.
4926    async fn try_recover_from_context_limit(
4927        &self,
4928        thread_id: &ThreadId,
4929        turn_id: &TurnId,
4930        provider: &str,
4931        model: &str,
4932        error: &str,
4933        transcript: &mut Vec<TranscriptItem>,
4934        compacted_this_turn: &mut bool,
4935        recovery_attempts: &mut u32,
4936    ) -> anyhow::Result<bool> {
4937        const MAX_CONTEXT_LIMIT_RECOVERY_ATTEMPTS: u32 = 2;
4938        if !crate::compaction::is_context_limit_failure_message(error) {
4939            return Ok(false);
4940        }
4941        if *recovery_attempts >= MAX_CONTEXT_LIMIT_RECOVERY_ATTEMPTS {
4942            return Ok(false);
4943        }
4944        *recovery_attempts = recovery_attempts.saturating_add(1);
4945
4946        let error_item = TranscriptItem::Error(ErrorRecord {
4947            message: error.to_string(),
4948        });
4949        self.persist_turn_item(thread_id, turn_id, &error_item)
4950            .await?;
4951        transcript.push(error_item);
4952
4953        // After the first failed recovery, drop the item ahead of the latest
4954        // user/error so a second compact can succeed when the suffix itself is
4955        // still oversized (e.g. a huge tool result right before the prompt).
4956        if *recovery_attempts > 1 {
4957            *transcript =
4958                crate::compaction::strip_last_message_before_error(std::mem::take(transcript));
4959        }
4960
4961        let force_options = crate::compaction::CompactionOptions {
4962            allow_repeat: true,
4963            force: true,
4964            hysteresis_baseline: None,
4965            preserve_hint: Some(
4966                "Recover from a provider context/prompt-length limit; preserve the active user goal and recent tool outcomes."
4967                    .to_string(),
4968            ),
4969        };
4970        let before_len = transcript.len();
4971        let tool_images = self.tool_images(provider, model);
4972        let before_tokens = crate::compaction::estimate_prompt_tokens(transcript, tool_images);
4973        *transcript = self
4974            .compact_transcript_if_needed(
4975                thread_id,
4976                turn_id,
4977                provider,
4978                model,
4979                std::mem::take(transcript),
4980                force_options,
4981            )
4982            .await?;
4983        let after_tokens = crate::compaction::estimate_prompt_tokens(transcript, tool_images);
4984        *compacted_this_turn = *compacted_this_turn
4985            || transcript
4986                .iter()
4987                .any(crate::compaction::is_compaction_boundary);
4988
4989        self.emit(RoderEvent::ReliabilityRetryRecorded(
4990            ReliabilityRetryRecorded {
4991                context: ReliabilityContext {
4992                    thread_id: thread_id.clone(),
4993                    turn_id: turn_id.clone(),
4994                    provider: Some(provider.to_string()),
4995                    model: Some(model.to_string()),
4996                    ..ReliabilityContext::default()
4997                },
4998                error_class: ReliabilityErrorClass::ProviderError,
4999                decision: ReliabilityRetryDecision::Retry,
5000                attempt: *recovery_attempts,
5001                max_attempts: MAX_CONTEXT_LIMIT_RECOVERY_ATTEMPTS,
5002                delay_ms: Some(0),
5003                details: ReliabilityDetails::redacted(format!(
5004                    "context_limit_recovery: compact {before_tokens}->{after_tokens} tokens (items {before_len}->{}); {error}",
5005                    transcript.len()
5006                )),
5007                timestamp: OffsetDateTime::now_utc(),
5008            },
5009        ))
5010        .await;
5011
5012        // If force-compact did not shrink anything and we still have budget,
5013        // strip once more so the next round is not identical.
5014        if after_tokens >= before_tokens && *recovery_attempts < MAX_CONTEXT_LIMIT_RECOVERY_ATTEMPTS
5015        {
5016            *transcript =
5017                crate::compaction::strip_last_message_before_error(std::mem::take(transcript));
5018        }
5019        Ok(true)
5020    }
5021
5022    async fn fail_turn_with_error(
5023        &self,
5024        thread_id: &ThreadId,
5025        turn_id: &TurnId,
5026        message: String,
5027    ) -> anyhow::Result<()> {
5028        self.persist_turn_item(
5029            thread_id,
5030            turn_id,
5031            &TranscriptItem::Error(ErrorRecord {
5032                message: message.clone(),
5033            }),
5034        )
5035        .await?;
5036        self.emit(RoderEvent::TurnFailed(TurnFailed {
5037            thread_id: thread_id.clone(),
5038            turn_id: turn_id.clone(),
5039            error: message.clone(),
5040            error_kind: None,
5041            usage: None,
5042            timestamp: OffsetDateTime::now_utc(),
5043        }))
5044        .await;
5045        self.complete_team_member_turn_with_result(
5046            thread_id,
5047            turn_id,
5048            TeamMemberStatus::Failed,
5049            None,
5050            Some(message),
5051        )
5052        .await?;
5053        Ok(())
5054    }
5055
5056    async fn fail_turn_due_to_deadline(
5057        &self,
5058        thread_id: &ThreadId,
5059        turn_id: &TurnId,
5060        deadline: OffsetDateTime,
5061        transcript: &[TranscriptItem],
5062    ) -> anyhow::Result<()> {
5063        let partial_result = turn_partial_result(transcript);
5064        self.emit(RoderEvent::TurnPartialResult(TurnPartialResult {
5065            thread_id: thread_id.clone(),
5066            turn_id: turn_id.clone(),
5067            summary: partial_result.clone(),
5068            timestamp: OffsetDateTime::now_utc(),
5069        }))
5070        .await;
5071        self.emit(RoderEvent::TurnDeadlineExceeded(TurnDeadlineExceeded {
5072            thread_id: thread_id.clone(),
5073            turn_id: turn_id.clone(),
5074            deadline,
5075            partial_result: partial_result.clone(),
5076            timestamp: OffsetDateTime::now_utc(),
5077        }))
5078        .await;
5079        let message = "turn deadline expired".to_string();
5080        self.persist_turn_item(
5081            thread_id,
5082            turn_id,
5083            &TranscriptItem::Error(ErrorRecord {
5084                message: format!("{message}: {partial_result}"),
5085            }),
5086        )
5087        .await?;
5088        self.emit(RoderEvent::TurnFailed(TurnFailed {
5089            thread_id: thread_id.clone(),
5090            turn_id: turn_id.clone(),
5091            error: message.clone(),
5092            error_kind: Some("deadline_timeout".to_string()),
5093            usage: None,
5094            timestamp: OffsetDateTime::now_utc(),
5095        }))
5096        .await;
5097        self.complete_team_member_turn_with_result(
5098            thread_id,
5099            turn_id,
5100            TeamMemberStatus::Failed,
5101            None,
5102            Some(format!("{message}: {partial_result}")),
5103        )
5104        .await?;
5105        Ok(())
5106    }
5107
5108    async fn start_deadline_finalization(
5109        &self,
5110        thread_id: &ThreadId,
5111        turn_id: &TurnId,
5112        transcript: &mut Vec<TranscriptItem>,
5113        remaining_seconds: u64,
5114    ) -> anyhow::Result<()> {
5115        let item = TranscriptItem::UserMessage(crate::deadline_policy::finalization_message(
5116            remaining_seconds,
5117        ));
5118        self.persist_turn_item(thread_id, turn_id, &item).await?;
5119        transcript.push(item);
5120        self.emit(RoderEvent::TurnPartialResult(TurnPartialResult {
5121            thread_id: thread_id.clone(),
5122            turn_id: turn_id.clone(),
5123            summary: turn_partial_result(transcript),
5124            timestamp: OffsetDateTime::now_utc(),
5125        }))
5126        .await;
5127        Ok(())
5128    }
5129
5130    /// Emits the reliability-limit event for a continuation (loop persistence)
5131    /// without failing the turn. The caller resets the failure counter and
5132    /// injects a nudge so the round loop keeps going.
5133    async fn record_reliability_limit_continuation(
5134        &self,
5135        thread_id: &ThreadId,
5136        turn_id: &TurnId,
5137        provider: &str,
5138        model: &str,
5139        limit: ReliabilityLimitHit,
5140    ) {
5141        self.emit(RoderEvent::ReliabilityLimitRecorded(
5142            ReliabilityLimitRecorded {
5143                context: ReliabilityContext {
5144                    thread_id: thread_id.clone(),
5145                    turn_id: turn_id.clone(),
5146                    tool_id: None,
5147                    tool_name: None,
5148                    provider: Some(provider.to_string()),
5149                    model: Some(model.to_string()),
5150                },
5151                error_class: limit.error_class,
5152                limit_kind: limit.limit_kind,
5153                decision: limit.decision,
5154                current: limit.current,
5155                limit: limit.limit,
5156                details: ReliabilityDetails::redacted(&limit.message),
5157                timestamp: OffsetDateTime::now_utc(),
5158            },
5159        ))
5160        .await;
5161    }
5162
5163    async fn fail_turn_due_to_reliability_limit(
5164        &self,
5165        thread_id: &ThreadId,
5166        turn_id: &TurnId,
5167        provider: &str,
5168        model: &str,
5169        limit: ReliabilityLimitHit,
5170        transcript: &[TranscriptItem],
5171    ) -> anyhow::Result<()> {
5172        self.emit(RoderEvent::ReliabilityLimitRecorded(
5173            ReliabilityLimitRecorded {
5174                context: ReliabilityContext {
5175                    thread_id: thread_id.clone(),
5176                    turn_id: turn_id.clone(),
5177                    tool_id: None,
5178                    tool_name: None,
5179                    provider: Some(provider.to_string()),
5180                    model: Some(model.to_string()),
5181                },
5182                error_class: limit.error_class,
5183                limit_kind: limit.limit_kind,
5184                decision: limit.decision,
5185                current: limit.current,
5186                limit: limit.limit,
5187                details: ReliabilityDetails::redacted(&limit.message),
5188                timestamp: OffsetDateTime::now_utc(),
5189            },
5190        ))
5191        .await;
5192        let partial_result = turn_partial_result(transcript);
5193        self.emit(RoderEvent::TurnPartialResult(TurnPartialResult {
5194            thread_id: thread_id.clone(),
5195            turn_id: turn_id.clone(),
5196            summary: partial_result.clone(),
5197            timestamp: OffsetDateTime::now_utc(),
5198        }))
5199        .await;
5200        let message = format!("reliability limit reached: {}", limit.message);
5201        self.persist_turn_item(
5202            thread_id,
5203            turn_id,
5204            &TranscriptItem::Error(ErrorRecord {
5205                message: format!("{message}: {partial_result}"),
5206            }),
5207        )
5208        .await?;
5209        self.emit(RoderEvent::TurnFailed(TurnFailed {
5210            thread_id: thread_id.clone(),
5211            turn_id: turn_id.clone(),
5212            error: message.clone(),
5213            error_kind: Some("reliability_limit".to_string()),
5214            usage: None,
5215            timestamp: OffsetDateTime::now_utc(),
5216        }))
5217        .await;
5218        self.complete_team_member_turn_with_result(
5219            thread_id,
5220            turn_id,
5221            TeamMemberStatus::Failed,
5222            None,
5223            Some(format!("{message}: {partial_result}")),
5224        )
5225        .await?;
5226        Ok(())
5227    }
5228
5229    async fn append_steers(
5230        &self,
5231        req: &StartTurnRequest,
5232        turn_id: &TurnId,
5233        transcript: &mut Vec<TranscriptItem>,
5234        steers: Vec<QueuedTurnSteer>,
5235    ) -> anyhow::Result<()> {
5236        for queued in steers {
5237            let mut steer = queued.message;
5238            steer.text = steer.text.trim().to_string();
5239            if steer.text.is_empty() && steer.images.is_empty() {
5240                continue;
5241            }
5242            let item = TranscriptItem::UserMessage(steer);
5243            self.persist_turn_item(&req.thread_id, turn_id, &item)
5244                .await?;
5245            transcript.push(item);
5246            if let Some(ack) = queued.mailbox_ack {
5247                self.teams
5248                    .mark_mailbox_messages_delivered(&ack.team_id, turn_id, &ack.message_ids)
5249                    .await?;
5250            }
5251        }
5252        Ok(())
5253    }
5254
5255    async fn persist_model_profile_segment(
5256        &self,
5257        thread_id: &ThreadId,
5258        turn_id: &TurnId,
5259        profile: Option<&ModelHarnessProfile>,
5260        provider: &str,
5261        model: &str,
5262        segment: &str,
5263    ) -> anyhow::Result<()> {
5264        let item = TranscriptItem::ProviderMetadata(model_profile_segment_metadata(
5265            profile, provider, model, segment,
5266        ));
5267        self.persist_turn_item(thread_id, turn_id, &item).await
5268    }
5269
5270    fn task_ledger_tool_specs(
5271        &self,
5272        profile: Option<&ModelHarnessProfile>,
5273    ) -> Vec<roder_api::tools::ToolSpec> {
5274        self.tool_registry
5275            .get(TASK_LEDGER_TOOL_NAME)
5276            .map(|tool| {
5277                tool.spec()
5278                    .normalized_for_model_profile(schema_policy_for_model(profile))
5279            })
5280            .into_iter()
5281            .collect()
5282    }
5283
5284    pub(crate) fn engine_for(&self, provider: &str) -> anyhow::Result<Arc<dyn InferenceEngine>> {
5285        self.registry
5286            .inference_engine(provider)
5287            .or_else(|| {
5288                self.registry
5289                    .default_inference_engine()
5290                    .filter(|engine| provider.is_empty() || engine.id() == provider)
5291            })
5292            .ok_or_else(|| anyhow::anyhow!("inference provider {provider:?} is not registered"))
5293    }
5294
5295    pub async fn emit(&self, event: RoderEvent) -> EventEnvelope {
5296        self.dispatch_local_hook_lifecycle(&event).await;
5297        if let Some(record) = Self::lifecycle_record_for_event(&event) {
5298            let _ = self.persist_turn_lifecycle_record(&record).await;
5299        }
5300        let envelope = self.bus.emit(event);
5301        if let (Some(store), Some(thread_id)) = (&self.thread_store, envelope.thread_id.as_ref())
5302            && should_persist_thread_event(thread_id)
5303        {
5304            let _ = store.append_event(thread_id, &envelope).await;
5305        }
5306        self.dispatch_event_sinks(&envelope).await;
5307        envelope
5308    }
5309
5310    async fn dispatch_local_hook_lifecycle(&self, event: &RoderEvent) {
5311        // SessionStart belongs to the session, not the turn: firing it from
5312        // TurnStarted re-ran setup hooks on every user message. ThreadCreated
5313        // and ThreadLoaded carry no turn id, so they borrow a synthetic one the
5314        // way the SessionEnd path does.
5315        let session_turn_id = SESSION_HOOK_TURN_ID.to_string();
5316        let (thread_id, turn_id, name, matcher, input) = match event {
5317            RoderEvent::ThreadCreated(event) => (
5318                &event.thread_id,
5319                &session_turn_id,
5320                "SessionStart",
5321                Some("startup"),
5322                serde_json::json!({"source":"startup"}),
5323            ),
5324            RoderEvent::ThreadLoaded(event) => (
5325                &event.thread_id,
5326                &session_turn_id,
5327                "SessionStart",
5328                Some("resume"),
5329                serde_json::json!({"source":"resume"}),
5330            ),
5331            RoderEvent::TurnCompleted(event) => (
5332                &event.thread_id,
5333                &event.turn_id,
5334                "Stop",
5335                None,
5336                serde_json::json!({"finishReason":event.finish_reason}),
5337            ),
5338            RoderEvent::TurnFailed(event) => (
5339                &event.thread_id,
5340                &event.turn_id,
5341                "Stop",
5342                None,
5343                serde_json::json!({"error":event.error}),
5344            ),
5345            RoderEvent::ContextCompactionStarted(event) => (
5346                &event.thread_id,
5347                &event.turn_id,
5348                "PreCompact",
5349                None,
5350                serde_json::json!({"originalItemCount":event.original_item_count,"originalEstimatedTokens":event.original_estimated_tokens}),
5351            ),
5352            RoderEvent::ContextCompactionRecorded(event) => (
5353                &event.thread_id,
5354                &event.turn_id,
5355                "PostCompact",
5356                None,
5357                serde_json::json!({"compactedItemCount":event.compacted_item_count,"compactedEstimatedTokens":event.compacted_estimated_tokens}),
5358            ),
5359            RoderEvent::SubagentStarted(event) => (
5360                &event.parent_thread_id,
5361                &event.parent_turn_id,
5362                "SubagentStart",
5363                Some(event.agent_type.as_str()),
5364                serde_json::json!({"subagentThreadId":event.thread_id,"agentType":event.agent_type,"description":event.description}),
5365            ),
5366            RoderEvent::SubagentCompleted(event) => (
5367                &event.parent_thread_id,
5368                &event.parent_turn_id,
5369                "SubagentStop",
5370                Some(event.agent_type.as_str()),
5371                serde_json::json!({"subagentThreadId":event.thread_id,"agentType":event.agent_type,"exitReason":event.exit_reason}),
5372            ),
5373            RoderEvent::SubagentFailed(event) => (
5374                &event.parent_thread_id,
5375                &event.parent_turn_id,
5376                "SubagentStop",
5377                Some(event.agent_type.as_str()),
5378                serde_json::json!({"subagentThreadId":event.thread_id,"agentType":event.agent_type,"error":event.error}),
5379            ),
5380            _ => return,
5381        };
5382        // One SessionStart per session, whichever event opened it.
5383        if name == "SessionStart" && !self.claim_session_hook_start(thread_id).await {
5384            return;
5385        }
5386        let workspace = self.config.read().await.workspace.clone();
5387        crate::hooks::run_lifecycle(
5388            self,
5389            thread_id,
5390            turn_id,
5391            workspace.as_deref(),
5392            name,
5393            matcher,
5394            input,
5395        )
5396        .await;
5397    }
5398
5399    /// Claim the single `SessionStart` dispatch for a thread, returning whether
5400    /// this caller won it.
5401    pub(crate) async fn claim_session_hook_start(&self, thread_id: &ThreadId) -> bool {
5402        self.session_hook_started
5403            .write()
5404            .await
5405            .insert(thread_id.clone())
5406    }
5407
5408    /// Records a projected thread item event and persists it through the configured thread store.
5409    ///
5410    /// App-server protocol notification bridges currently call this after translating runtime
5411    /// events into item events. Headless runtime consumers that subscribe to `RoderEvent` directly
5412    /// must make the same call if they need the derived item event stream persisted.
5413    pub async fn record_thread_item_event_kind(
5414        &self,
5415        thread_id: &ThreadId,
5416        turn_id: &TurnId,
5417        timestamp: OffsetDateTime,
5418        kind: ThreadItemEventKind,
5419    ) -> anyhow::Result<ThreadItemEvent> {
5420        let seq = self.next_thread_item_event_seq(thread_id).await?;
5421        let item_event = ThreadItemEvent {
5422            seq,
5423            event_id: uuid::Uuid::new_v4().to_string(),
5424            thread_id: thread_id.clone(),
5425            turn_id: turn_id.clone(),
5426            timestamp,
5427            event: kind,
5428        };
5429        if let Some(store) = &self.thread_store {
5430            store.append_item_event(thread_id, &item_event).await?;
5431        }
5432        self.remember_thread_item_event(&item_event).await?;
5433        Ok(item_event)
5434    }
5435
5436    async fn next_thread_item_event_seq(&self, thread_id: &ThreadId) -> anyhow::Result<u64> {
5437        self.ensure_thread_item_cache(thread_id).await?;
5438        Ok(self
5439            .thread_item_cache
5440            .lock()
5441            .await
5442            .next_item_event_seq(thread_id))
5443    }
5444
5445    pub async fn thread_item_exists(
5446        &self,
5447        thread_id: &ThreadId,
5448        turn_id: &TurnId,
5449        item_id: &str,
5450    ) -> anyhow::Result<bool> {
5451        self.ensure_thread_item_cache(thread_id).await?;
5452        Ok(self
5453            .thread_item_cache
5454            .lock()
5455            .await
5456            .thread_item_exists(thread_id, turn_id, item_id))
5457    }
5458
5459    pub async fn current_reasoning_item_id(
5460        &self,
5461        thread_id: &ThreadId,
5462        turn_id: &TurnId,
5463    ) -> anyhow::Result<Option<String>> {
5464        self.ensure_thread_item_cache(thread_id).await?;
5465        Ok(self
5466            .thread_item_cache
5467            .lock()
5468            .await
5469            .current_reasoning_item_id(thread_id, turn_id))
5470    }
5471
5472    async fn remember_thread_item_event(&self, item_event: &ThreadItemEvent) -> anyhow::Result<()> {
5473        self.ensure_thread_item_cache(&item_event.thread_id).await?;
5474        self.thread_item_cache
5475            .lock()
5476            .await
5477            .remember_item_event(item_event);
5478        Ok(())
5479    }
5480
5481    pub async fn latest_transcript_item_index(
5482        &self,
5483        thread_id: &ThreadId,
5484        turn_id: &TurnId,
5485    ) -> anyhow::Result<Option<usize>> {
5486        self.ensure_thread_item_cache(thread_id).await?;
5487        Ok(self
5488            .thread_item_cache
5489            .lock()
5490            .await
5491            .latest_transcript_item_index(thread_id, turn_id))
5492    }
5493
5494    async fn next_transcript_item_index(
5495        &self,
5496        thread_id: &ThreadId,
5497        turn_id: &TurnId,
5498    ) -> anyhow::Result<usize> {
5499        self.ensure_thread_item_cache(thread_id).await?;
5500        Ok(self
5501            .thread_item_cache
5502            .lock()
5503            .await
5504            .next_transcript_item_index(thread_id, turn_id))
5505    }
5506
5507    async fn remember_transcript_item_index(
5508        &self,
5509        thread_id: &ThreadId,
5510        turn_id: &TurnId,
5511        item_index: usize,
5512    ) -> anyhow::Result<()> {
5513        self.ensure_thread_item_cache(thread_id).await?;
5514        self.thread_item_cache
5515            .lock()
5516            .await
5517            .remember_transcript_item_index(thread_id, turn_id, item_index);
5518        Ok(())
5519    }
5520
5521    async fn ensure_thread_item_cache(&self, thread_id: &ThreadId) -> anyhow::Result<()> {
5522        if self
5523            .thread_item_cache
5524            .lock()
5525            .await
5526            .contains_thread(thread_id)
5527        {
5528            return Ok(());
5529        }
5530
5531        let snapshot = if let Some(store) = &self.thread_store {
5532            store.load_thread(thread_id).await?
5533        } else {
5534            None
5535        };
5536        self.thread_item_cache.lock().await.ensure_thread(
5537            thread_id,
5538            ThreadItemCacheEntry::from_snapshot(snapshot.as_ref()),
5539        );
5540        Ok(())
5541    }
5542
5543    pub(crate) async fn persist_turn_item(
5544        &self,
5545        thread_id: &ThreadId,
5546        turn_id: &TurnId,
5547        item: &TranscriptItem,
5548    ) -> anyhow::Result<()> {
5549        let item_index = self.next_transcript_item_index(thread_id, turn_id).await?;
5550        let timestamp = OffsetDateTime::now_utc();
5551        self.emit(RoderEvent::TranscriptItemAppended(TranscriptItemAppended {
5552            thread_id: thread_id.clone(),
5553            turn_id: turn_id.clone(),
5554            item_type: match item {
5555                TranscriptItem::UserMessage(_) => "user_message",
5556                TranscriptItem::AssistantMessage(_) => "assistant_message",
5557                TranscriptItem::ReasoningSummary(_) => "reasoning_summary",
5558                TranscriptItem::ToolCall(_) => "tool_call",
5559                TranscriptItem::ToolResult(_) => "tool_result",
5560                TranscriptItem::FileChange(_) => "file_change",
5561                TranscriptItem::ContextCompaction(_) => "context_compaction",
5562                TranscriptItem::Error(_) => "error",
5563                TranscriptItem::ProviderMetadata(_) => "provider_metadata",
5564            }
5565            .to_string(),
5566            item_index: Some(item_index),
5567            item: Some(item.clone()),
5568            timestamp,
5569        }))
5570        .await;
5571        self.remember_transcript_item_index(thread_id, turn_id, item_index)
5572            .await?;
5573        Ok(())
5574    }
5575}
5576
5577fn transcript_has_images(transcript: &[TranscriptItem]) -> bool {
5578    transcript.iter().any(|item| {
5579        matches!(
5580            item,
5581            TranscriptItem::UserMessage(message) if !message.images.is_empty()
5582        )
5583    })
5584}
5585
5586fn transcript_has_task_ledger(transcript: &[TranscriptItem]) -> bool {
5587    transcript.iter().any(|item| {
5588        matches!(
5589            item,
5590            TranscriptItem::ToolResult(result)
5591                if result.name.as_deref() == Some(TASK_LEDGER_TOOL_NAME) && !result.is_error
5592        )
5593    })
5594}
5595
5596fn task_ledger_completion_prompt(transcript: &[TranscriptItem]) -> Option<String> {
5597    let latest = transcript.iter().rev().find_map(|item| match item {
5598        TranscriptItem::ToolResult(result)
5599            if result.name.as_deref() == Some(TASK_LEDGER_TOOL_NAME) && !result.is_error =>
5600        {
5601            Some(result.result.as_str())
5602        }
5603        _ => None,
5604    })?;
5605    if !task_ledger_has_open_items(latest) {
5606        return None;
5607    }
5608
5609    let mut ledger = latest.chars().take(1500).collect::<String>();
5610    if latest.chars().nth(1500).is_some() {
5611        ledger.push_str("...");
5612    }
5613    Some(format!(
5614        "Task Ledger Completion Required: the latest task ledger still has pending or in-progress items. Do not provide a final answer yet. Use tools to complete the remaining scoreable work, create or update any required output files, then call `{TASK_LEDGER_TOOL_NAME}` with every task completed and evidence before finalizing.\n\nLatest ledger:\n{ledger}"
5615    ))
5616}
5617
5618fn task_ledger_deadline_completion_prompt(
5619    remaining_seconds: u64,
5620    reserve_seconds: u64,
5621    completion_prompt: &str,
5622) -> String {
5623    format!(
5624        "Eval deadline scoreable completion: {remaining_seconds} seconds remain in the {reserve_seconds}-second finalization reserve. Do not browse, search, or start slow work. Use the available tools now to create or update the required scoreable output files, run only a quick local check if needed, then update the task ledger to completed before finalizing.\n\n{completion_prompt}"
5625    )
5626}
5627
5628fn task_ledger_scoreable_checkpoint_prompt(
5629    remaining_seconds: u64,
5630    completion_prompt: &str,
5631) -> String {
5632    format!(
5633        "Scoreable Output Checkpoint: {remaining_seconds} seconds remain before the eval deadline. Before any further research, browsing, or long commands, use tools now to ensure the required output file(s) exist with the best evidence-backed answer, even if provisional. If a scoreable file already exists, read it and preserve that candidate unless you have stronger task-specific evidence for a replacement. Do not overwrite a plausible dated, historical, or local-evidence candidate with a current live-page, partial-coverage, or weaker guess merely to refresh the checkpoint. You may continue refining afterward, but do not apologize or finalize until the scoreable file exists and the task ledger is updated.\n\n{completion_prompt}"
5634    )
5635}
5636
5637fn task_ledger_has_open_items(ledger: &str) -> bool {
5638    ledger.lines().any(|line| {
5639        let line = line.trim_start();
5640        line.starts_with("- pending:") || line.starts_with("- in_progress:")
5641    })
5642}
5643
5644fn turn_deadline_for_config(cfg: &RuntimeConfig) -> Option<OffsetDateTime> {
5645    cfg.turn_deadline_seconds
5646        .filter(|seconds| *seconds > 0)
5647        .map(|seconds| OffsetDateTime::now_utc() + Duration::seconds(seconds as i64))
5648}
5649
5650fn deadline_expired(deadline: OffsetDateTime) -> bool {
5651    OffsetDateTime::now_utc() >= deadline
5652}
5653
5654pub(crate) fn deadline_remaining_seconds(deadline: Option<OffsetDateTime>) -> Option<u64> {
5655    let deadline = deadline?;
5656    if deadline <= OffsetDateTime::now_utc() {
5657        return Some(0);
5658    }
5659    Some(
5660        (deadline - OffsetDateTime::now_utc())
5661            .unsigned_abs()
5662            .as_secs()
5663            .max(1),
5664    )
5665}
5666
5667fn deadline_instant(deadline: OffsetDateTime) -> tokio::time::Instant {
5668    let now = OffsetDateTime::now_utc();
5669    if deadline <= now {
5670        return tokio::time::Instant::now();
5671    }
5672    tokio::time::Instant::now() + (deadline - now).unsigned_abs()
5673}
5674
5675fn inference_timeout_deadline(
5676    deadline: Option<OffsetDateTime>,
5677    runtime_profile: RuntimeProfile,
5678    task_ledger_required: bool,
5679    reserve_seconds: u64,
5680    finalization_requested: bool,
5681    task_ledger_scoreable_checkpoints: u8,
5682    transcript: &[TranscriptItem],
5683) -> Option<(OffsetDateTime, InferenceTimeoutAction)> {
5684    let deadline = deadline?;
5685    if runtime_profile == RuntimeProfile::Eval
5686        && task_ledger_required
5687        && !finalization_requested
5688        && task_ledger_scoreable_checkpoints < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
5689        && TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS > reserve_seconds
5690        && task_ledger_completion_prompt(transcript).is_some()
5691    {
5692        let checkpoint_deadline =
5693            deadline - Duration::seconds(TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS as i64);
5694        if checkpoint_deadline > OffsetDateTime::now_utc() {
5695            return Some((
5696                checkpoint_deadline,
5697                InferenceTimeoutAction::ScoreableCheckpoint,
5698            ));
5699        }
5700    }
5701    if runtime_profile == RuntimeProfile::Eval && !finalization_requested {
5702        return Some((
5703            deadline - Duration::seconds(reserve_seconds as i64),
5704            InferenceTimeoutAction::Finalization,
5705        ));
5706    }
5707    Some((deadline, InferenceTimeoutAction::Finalization))
5708}
5709
5710fn turn_partial_result(transcript: &[TranscriptItem]) -> String {
5711    let tool_results = transcript
5712        .iter()
5713        .filter(|item| matches!(item, TranscriptItem::ToolResult(_)))
5714        .count();
5715    let assistant_messages = transcript
5716        .iter()
5717        .filter(|item| matches!(item, TranscriptItem::AssistantMessage(_)))
5718        .count();
5719    format!(
5720        "partial turn state: {} transcript items, {assistant_messages} assistant messages, {tool_results} tool results",
5721        transcript.len()
5722    )
5723}
5724
5725fn reasoning_for_model(cfg: &RuntimeConfig, model: &str) -> ReasoningConfig {
5726    let level = effective_reasoning_for_model(cfg, model);
5727    match level.as_str() {
5728        "" | REASONING_NONE => ReasoningConfig::default(),
5729        level => ReasoningConfig {
5730            enabled: true,
5731            level: Some(level.to_string()),
5732        },
5733    }
5734}
5735
5736fn server_side_compaction_threshold(cfg: &RuntimeConfig, model: &str) -> Option<u32> {
5737    let entry = lookup_model(model)?;
5738    if !entry.supports_compaction {
5739        return None;
5740    }
5741    cfg.auto_compact_token_limit
5742        .or_else(|| {
5743            model_profile_for_model(cfg, model).and_then(|profile| profile.auto_compact_token_limit)
5744        })
5745        .or(Some(entry.auto_compact_token_limit))
5746        .filter(|threshold| *threshold > 0)
5747}
5748
5749pub(crate) fn tool_search_for_provider_model(
5750    cfg: &RuntimeConfig,
5751    provider: &str,
5752    model: &str,
5753) -> ToolSearchConfig {
5754    let mut resolved = cfg.tool_search.clone();
5755    if let Some(provider_config) = cfg.provider_tool_search.get(provider) {
5756        provider_config.apply_to(&mut resolved);
5757    }
5758    if let Some(model_config) = cfg.model_tool_search.get(model) {
5759        model_config.apply_to(&mut resolved);
5760    }
5761    resolved
5762}
5763
5764fn parallel_tool_calls_for_model(cfg: &RuntimeConfig, model: &str) -> bool {
5765    cfg.model_parallel_tool_calls
5766        .get(model)
5767        .copied()
5768        .or_else(|| {
5769            model_profile_for_model(cfg, model).and_then(|profile| profile.parallel_tool_calls)
5770        })
5771        .unwrap_or(true)
5772}
5773
5774/// Count the children an `agent_swarm` call will launch from its arguments
5775/// (item-based spawns plus resumes). Lenient: returns 0 on malformed input.
5776fn agent_swarm_child_count(arguments: &str) -> usize {
5777    serde_json::from_str::<roder_api::subagents::AgentSwarmRequest>(arguments)
5778        .map(|request| request.items.len() + request.resume_agent_ids.len())
5779        .unwrap_or(0)
5780}
5781
5782/// Parse `completed`/`failed`/`aborted` counts from an `<agent_swarm_result>`
5783/// summary line, e.g. `<summary>completed: 2, failed: 1</summary>`. Omitted
5784/// buckets default to 0. Returns `None` when the text is not a swarm result.
5785fn parse_swarm_counts(text: &str) -> Option<(usize, usize, usize)> {
5786    if !text.contains("<agent_swarm_result>") {
5787        return None;
5788    }
5789    let bucket = |label: &str| -> usize {
5790        let needle = format!("{label}: ");
5791        text.find(&needle)
5792            .map(|start| start + needle.len())
5793            .map(|start| {
5794                text[start..]
5795                    .chars()
5796                    .take_while(|c| c.is_ascii_digit())
5797                    .collect::<String>()
5798            })
5799            .and_then(|digits| digits.parse().ok())
5800            .unwrap_or(0)
5801    };
5802    Some((bucket("completed"), bucket("failed"), bucket("aborted")))
5803}
5804
5805fn effective_reasoning_for_model(cfg: &RuntimeConfig, model: &str) -> String {
5806    let base_reasoning = default_effective_reasoning_for_model(cfg, model);
5807    if cfg.dynamic_workflows.effort_profile == DynamicWorkflowEffortProfile::Ultracode {
5808        return ultracode_reasoning_level_for_model(
5809            model,
5810            &cfg.speed_policy.ultracode_reasoning,
5811            &base_reasoning,
5812        );
5813    }
5814    base_reasoning
5815}
5816
5817fn default_effective_reasoning_for_model(cfg: &RuntimeConfig, model: &str) -> String {
5818    let Some(entry) = lookup_model(model) else {
5819        return cfg
5820            .reasoning
5821            .clone()
5822            .unwrap_or_else(|| REASONING_NONE.to_string());
5823    };
5824    if entry.supported_reasoning.is_empty() {
5825        return REASONING_NONE.to_string();
5826    }
5827    cfg.reasoning
5828        .as_deref()
5829        .filter(|reasoning| {
5830            entry
5831                .supported_reasoning
5832                .iter()
5833                .any(|option| option.effort == *reasoning)
5834        })
5835        .map(str::to_string)
5836        .or_else(|| {
5837            model_profile_for_model(cfg, model)
5838                .and_then(|profile| profile.reasoning.orientation)
5839                .filter(|reasoning| {
5840                    entry
5841                        .supported_reasoning
5842                        .iter()
5843                        .any(|option| option.effort == reasoning)
5844                })
5845        })
5846        .unwrap_or_else(|| entry.default_reasoning.to_string())
5847}
5848
5849fn validate_reasoning_effort(model: &str, effort: &str) -> anyhow::Result<()> {
5850    if effort == REASONING_NONE && !model_supports_reasoning(model, effort) {
5851        return Ok(());
5852    }
5853    let Some(entry) = lookup_model(model) else {
5854        return Ok(());
5855    };
5856    if entry
5857        .supported_reasoning
5858        .iter()
5859        .any(|option| option.effort == effort)
5860    {
5861        Ok(())
5862    } else {
5863        anyhow::bail!("model {model} does not support reasoning effort {effort}")
5864    }
5865}
5866
5867fn validate_runtime_config_reasoning(cfg: &RuntimeConfig) -> anyhow::Result<()> {
5868    let Some(reasoning) = cfg.reasoning.as_deref() else {
5869        return Ok(());
5870    };
5871    let Some(entry) = lookup_model(&cfg.default_model) else {
5872        return Ok(());
5873    };
5874    if entry.provider != PROVIDER_GEMINI {
5875        return Ok(());
5876    }
5877    validate_reasoning_effort(&cfg.default_model, reasoning)
5878}
5879
5880fn validate_runtime_inference_router_config(
5881    registry: &ExtensionRegistry,
5882    cfg: &RuntimeConfig,
5883) -> anyhow::Result<()> {
5884    if !cfg.inference_router.enabled {
5885        return Ok(());
5886    }
5887    let Some(router_id) = cfg.inference_router.router_id.as_deref() else {
5888        anyhow::bail!("inference_router.enabled requires inference_router.router");
5889    };
5890    if registry.inference_router(router_id).is_some() {
5891        return Ok(());
5892    }
5893    let available = registry
5894        .inference_routers
5895        .iter()
5896        .map(|router| router.id())
5897        .collect::<Vec<_>>()
5898        .join(", ");
5899    if available.is_empty() {
5900        anyhow::bail!("inference router {router_id:?} is not registered");
5901    }
5902    anyhow::bail!(
5903        "inference router {router_id:?} is not registered; available routers: {available}"
5904    );
5905}
5906
5907fn model_supports_reasoning(model: &str, effort: &str) -> bool {
5908    lookup_model(model)
5909        .map(|entry| {
5910            entry
5911                .supported_reasoning
5912                .iter()
5913                .any(|option| option.effort == effort)
5914        })
5915        .unwrap_or(false)
5916}
5917
5918fn is_final_answer_phase(phase: Option<&str>) -> bool {
5919    phase.is_none_or(|phase| phase.is_empty() || phase == FINAL_ANSWER_PHASE)
5920}
5921
5922fn edit_tool_for_model<'a>(cfg: &'a RuntimeConfig, model: &'a str) -> Option<&'a str> {
5923    cfg.model_edit_tools
5924        .get(model)
5925        .map(String::as_str)
5926        .or_else(|| {
5927            cfg.model_profiles
5928                .get(model)
5929                .and_then(|profile| profile.edit_tool.as_deref())
5930        })
5931        .or_else(|| lookup_model(model).and_then(|entry| entry.edit_tool))
5932        .or(Some(EDIT_TOOL_EDIT))
5933}
5934
5935fn model_profile_for_model(cfg: &RuntimeConfig, model: &str) -> Option<ModelHarnessProfile> {
5936    cfg.model_profiles
5937        .get(model)
5938        .cloned()
5939        .or_else(|| built_in_model_profile(model))
5940}
5941
5942pub(crate) fn allowlist_permits(allowlist: &[String], tool_name: &str) -> bool {
5943    allowlist.is_empty() || allowlist.iter().any(|allowed| allowed == tool_name)
5944}
5945
5946/// Provider-aware profile resolution for the active turn.
5947///
5948/// Many model ids are shared across providers (for example Cursor proxies
5949/// `claude-opus-4-8`). Resolving the harness profile by id alone picks the
5950/// first catalog entry, which assigns cross-provider ids the wrong family and
5951/// instruction overlay (e.g. a `cursor/claude-opus-4-8` turn would otherwise
5952/// inherit the Anthropic overlay). Prefer the explicit `(provider, model)`
5953/// catalog entry, keeping user-configured profile overrides as the top
5954/// precedence.
5955fn model_profile_for_provider_model(
5956    cfg: &RuntimeConfig,
5957    provider: &str,
5958    model: &str,
5959) -> Option<ModelHarnessProfile> {
5960    cfg.model_profiles
5961        .get(model)
5962        .cloned()
5963        .or_else(|| built_in_model_profile_for_provider(provider, model))
5964}
5965
5966fn schema_policy_for_model(profile: Option<&ModelHarnessProfile>) -> ModelSchemaPolicy {
5967    profile
5968        .map(|profile| profile.schema_policy)
5969        .unwrap_or_default()
5970}
5971
5972fn model_profile_segment_metadata(
5973    profile: Option<&ModelHarnessProfile>,
5974    provider: &str,
5975    model: &str,
5976    segment: &str,
5977) -> serde_json::Value {
5978    serde_json::json!({
5979        "kind": MODEL_PROFILE_TRACE_KIND,
5980        "segment": segment,
5981        "provider": provider,
5982        "model": model,
5983        "profileModel": profile.map(|profile| profile.model.as_str()).unwrap_or(model),
5984        "providerFamily": profile.map(|profile| profile.provider_family),
5985        "editTool": profile.and_then(|profile| profile.edit_tool.as_deref()),
5986        "schemaPolicy": profile.map(|profile| profile.schema_policy),
5987        "instructionOverlay": profile.map(|profile| profile.instruction_overlay),
5988        "parallelToolCalls": profile.and_then(|profile| profile.parallel_tool_calls),
5989        "autoCompactTokenLimit": profile.and_then(|profile| profile.auto_compact_token_limit),
5990    })
5991}
5992
5993fn model_switch_summary(
5994    transcript: &[TranscriptItem],
5995    profile: Option<&ModelHarnessProfile>,
5996    provider: &str,
5997    model: &str,
5998    tools: &[roder_api::tools::ToolSpec],
5999) -> Option<String> {
6000    let previous = latest_model_profile_segment(transcript)?;
6001    let previous_model = previous
6002        .get("model")
6003        .and_then(serde_json::Value::as_str)
6004        .unwrap_or_default();
6005    let previous_provider = previous
6006        .get("provider")
6007        .and_then(serde_json::Value::as_str)
6008        .unwrap_or_default();
6009    if previous_model == model && previous_provider == provider {
6010        return None;
6011    }
6012
6013    let previous_profile = previous
6014        .get("profileModel")
6015        .and_then(serde_json::Value::as_str)
6016        .unwrap_or(previous_model);
6017    let current_profile = profile
6018        .map(|profile| profile.model.as_str())
6019        .unwrap_or(model);
6020    let previous_edit_tool = previous
6021        .get("editTool")
6022        .and_then(serde_json::Value::as_str)
6023        .unwrap_or("none");
6024    let current_edit_tool = profile
6025        .and_then(|profile| profile.edit_tool.as_deref())
6026        .unwrap_or("none");
6027    let tool_names = tools
6028        .iter()
6029        .map(|tool| tool.name.as_str())
6030        .take(12)
6031        .collect::<Vec<_>>()
6032        .join(", ");
6033    Some(format!(
6034        "{MODEL_SWITCH_SUMMARY_PREFIX} previous profile {previous_provider}/{previous_profile} used edit tool {previous_edit_tool}. Current profile {provider}/{current_profile} uses edit tool {current_edit_tool}. Available tools now: {}.",
6035        if tool_names.is_empty() {
6036            "none"
6037        } else {
6038            &tool_names
6039        }
6040    ))
6041}
6042
6043fn latest_model_profile_segment(transcript: &[TranscriptItem]) -> Option<&serde_json::Value> {
6044    transcript.iter().rev().find_map(|item| {
6045        let TranscriptItem::ProviderMetadata(value) = item else {
6046            return None;
6047        };
6048        (value.get("kind").and_then(serde_json::Value::as_str) == Some(MODEL_PROFILE_TRACE_KIND))
6049            .then_some(value)
6050    })
6051}
6052
6053pub fn validate_edit_tool(value: &str) -> anyhow::Result<()> {
6054    match value.trim() {
6055        EDIT_TOOL_PATCH | EDIT_TOOL_EDIT => Ok(()),
6056        _ => anyhow::bail!(
6057            "unsupported edit_tool {value:?}; allowed values: {EDIT_TOOL_PATCH}, {EDIT_TOOL_EDIT}"
6058        ),
6059    }
6060}
6061
6062fn should_persist_thread_event(thread_id: &str) -> bool {
6063    !is_synthetic_event_thread_id(thread_id)
6064}
6065
6066#[cfg(test)]
6067#[path = "runtime/item_identity_tests.rs"]
6068mod item_identity_tests;
6069
6070#[cfg(test)]
6071#[path = "runtime/codex_v2_tests.rs"]
6072mod codex_v2_tests;
6073
6074#[cfg(test)]
6075#[path = "runtime/codex_v2_lifecycle_tests.rs"]
6076mod codex_v2_lifecycle_tests;
6077
6078#[cfg(test)]
6079mod tests {
6080    use super::*;
6081    use futures::stream;
6082    use roder_api::catalog::{
6083        PROVIDER_MOCK, REASONING_HIGH, REASONING_LOW, REASONING_MEDIUM, REASONING_MINIMAL,
6084        REASONING_NONE, REASONING_XHIGH,
6085    };
6086    use roder_api::extension::ExtensionRegistryBuilder;
6087    use roder_api::inference::{
6088        CompletionMetadata, InferenceCapabilities, InferenceEngine, InferenceEventStream,
6089        InferenceProviderContext, InferenceTurnContext, MessageDelta, ModelDescriptor,
6090        ModelInstructionOverlay, ModelProfileReasoning, ModelSchemaPolicy, ProviderFamily,
6091        ReasoningEffortDescriptor,
6092    };
6093    use roder_api::inference_routing::{
6094        InferenceRouter, InferenceRoutingContext, InferenceRoutingDecision, InferenceRoutingOutcome,
6095    };
6096    use roder_api::thread::ThreadStoreFactory;
6097    use roder_api::tools::{ToolContributor, ToolExecutor, ToolSpec};
6098    use roder_ext_jsonl_thread_store::store::JsonlThreadStoreFactory;
6099    use std::sync::Mutex as StdMutex;
6100
6101    fn test_workspace() -> String {
6102        std::env::current_dir().unwrap().display().to_string()
6103    }
6104
6105    #[test]
6106    fn interactive_explicit_turn_deadline_has_bounded_remaining_time() {
6107        let config = RuntimeConfig {
6108            runtime_profile: RuntimeProfile::Interactive,
6109            turn_deadline_seconds: Some(60),
6110            ..RuntimeConfig::default()
6111        };
6112
6113        let deadline = turn_deadline_for_config(&config).expect("configured turn deadline");
6114        let remaining = deadline_remaining_seconds(Some(deadline));
6115
6116        assert!(
6117            matches!(remaining, Some(1..=60)),
6118            "interactive deadline must be positive and no longer than configured: {remaining:?}"
6119        );
6120    }
6121
6122    #[test]
6123    fn interactive_without_turn_deadline_stays_unbounded() {
6124        let config = RuntimeConfig {
6125            runtime_profile: RuntimeProfile::Interactive,
6126            turn_deadline_seconds: None,
6127            ..RuntimeConfig::default()
6128        };
6129
6130        assert_eq!(turn_deadline_for_config(&config), None);
6131    }
6132
6133    struct MetadataMissingStore;
6134
6135    #[async_trait::async_trait]
6136    impl ThreadStore for MetadataMissingStore {
6137        fn id(&self) -> roder_api::thread::ThreadStoreId {
6138            "metadata-missing-store".to_string()
6139        }
6140
6141        async fn create_thread(&self, metadata: ThreadMetadata) -> anyhow::Result<ThreadMetadata> {
6142            Ok(metadata)
6143        }
6144
6145        async fn list_threads(&self) -> anyhow::Result<Vec<ThreadMetadata>> {
6146            Ok(Vec::new())
6147        }
6148
6149        async fn load_thread(
6150            &self,
6151            _thread_id: &ThreadId,
6152        ) -> anyhow::Result<Option<ThreadSnapshot>> {
6153            Ok(Some(ThreadSnapshot {
6154                metadata: None,
6155                ..ThreadSnapshot::default()
6156            }))
6157        }
6158
6159        async fn append_event(
6160            &self,
6161            _thread_id: &ThreadId,
6162            _envelope: &EventEnvelope,
6163        ) -> anyhow::Result<()> {
6164            Ok(())
6165        }
6166    }
6167
6168    struct MetadataMissingStoreFactory;
6169
6170    impl ThreadStoreFactory for MetadataMissingStoreFactory {
6171        fn id(&self) -> roder_api::thread::ThreadStoreId {
6172            "metadata-missing-store".to_string()
6173        }
6174
6175        fn create(&self) -> Arc<dyn ThreadStore> {
6176            Arc::new(MetadataMissingStore)
6177        }
6178    }
6179
6180    #[test]
6181    fn synthetic_app_server_events_are_not_thread_events() {
6182        for thread_id in ["app-server", "runtime", "thread-workflow"] {
6183            assert!(!should_persist_thread_event(thread_id));
6184        }
6185        assert!(should_persist_thread_event("thread-discovery"));
6186        assert!(should_persist_thread_event("thread-plan"));
6187        assert!(should_persist_thread_event("thread-process"));
6188        assert!(should_persist_thread_event("thread-1"));
6189    }
6190
6191    #[test]
6192    fn server_side_compaction_uses_catalog_ninety_percent_default() {
6193        assert_eq!(
6194            server_side_compaction_threshold(&RuntimeConfig::default(), "gpt-5.5"),
6195            Some(945_000)
6196        );
6197        assert_eq!(
6198            server_side_compaction_threshold(&RuntimeConfig::default(), "gpt-5.3-codex-spark"),
6199            Some(115_200)
6200        );
6201    }
6202
6203    #[test]
6204    fn server_side_compaction_respects_explicit_config_override() {
6205        let cfg = RuntimeConfig {
6206            auto_compact_token_limit: Some(123_456),
6207            ..RuntimeConfig::default()
6208        };
6209
6210        assert_eq!(
6211            server_side_compaction_threshold(&cfg, "gpt-5.5"),
6212            Some(123_456)
6213        );
6214    }
6215
6216    #[tokio::test]
6217    async fn pre_request_compaction_runs_when_server_side_model_is_at_context_window() {
6218        let captured = Arc::new(StdMutex::new(None));
6219        let mut builder = ExtensionRegistryBuilder::new();
6220        builder.inference_engine(Arc::new(CapturingEngine {
6221            request: captured.clone(),
6222        }));
6223        let thread_root = std::env::temp_dir().join(format!(
6224            "roder-pre-request-compaction-{}",
6225            uuid::Uuid::new_v4()
6226        ));
6227        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
6228            base_path: thread_root.clone(),
6229        }));
6230        let runtime = Arc::new(
6231            Runtime::new(
6232                builder.build().unwrap(),
6233                RuntimeConfig {
6234                    default_provider: PROVIDER_MOCK.to_string(),
6235                    default_model: "gpt-5.5".to_string(),
6236                    file_backed_dynamic_context: true,
6237                    ..RuntimeConfig::default()
6238                },
6239            )
6240            .unwrap(),
6241        );
6242        let thread_id = runtime
6243            .create_thread(Some("Pre-request compaction".to_string()))
6244            .await
6245            .unwrap()
6246            .thread_id;
6247        let old_turn = "old-turn".to_string();
6248        runtime
6249            .persist_turn_item(
6250                &thread_id,
6251                &old_turn,
6252                &TranscriptItem::UserMessage(UserMessage::text("old context ".repeat(4_300_000))),
6253            )
6254            .await
6255            .unwrap();
6256
6257        let mut events = runtime.subscribe_events();
6258        runtime
6259            .start_turn(StartTurnRequest {
6260                thread_id: thread_id.clone(),
6261                message: "continue".to_string(),
6262                images: Vec::new(),
6263                provider_override: None,
6264                model_override: None,
6265                reasoning_override: None,
6266                workspace: test_workspace(),
6267                instructions: InstructionBundle::default(),
6268                developer_context: None,
6269                task_ledger_required: false,
6270                service_tier_override: None,
6271            })
6272            .await
6273            .unwrap();
6274        loop {
6275            let envelope = tokio::time::timeout(std::time::Duration::from_secs(5), events.recv())
6276                .await
6277                .unwrap()
6278                .unwrap();
6279            if envelope.thread_id.as_deref() == Some(&thread_id)
6280                && matches!(envelope.event, RoderEvent::TurnCompleted(_))
6281            {
6282                break;
6283            }
6284        }
6285
6286        let request = captured.lock().unwrap().clone().unwrap();
6287        assert!(
6288            matches!(
6289                request.transcript.first(),
6290                Some(TranscriptItem::ContextCompaction(_))
6291            ),
6292            "provider request should start with a local emergency compaction item"
6293        );
6294        assert!(
6295            request.transcript.len() < 4,
6296            "provider request should not replay the full oversized prior transcript: {:?}",
6297            request.transcript
6298        );
6299
6300        let _ = std::fs::remove_dir_all(thread_root);
6301    }
6302
6303    #[tokio::test]
6304    async fn continue_after_context_window_failure_compacts_before_provider_request() {
6305        let captured = Arc::new(StdMutex::new(None));
6306        let mut builder = ExtensionRegistryBuilder::new();
6307        builder.inference_engine(Arc::new(CapturingEngine {
6308            request: captured.clone(),
6309        }));
6310        let thread_root = std::env::temp_dir().join(format!(
6311            "roder-context-failure-continue-{}",
6312            uuid::Uuid::new_v4()
6313        ));
6314        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
6315            base_path: thread_root.clone(),
6316        }));
6317        let runtime = Arc::new(
6318            Runtime::new(
6319                builder.build().unwrap(),
6320                RuntimeConfig {
6321                    default_provider: PROVIDER_MOCK.to_string(),
6322                    default_model: "gpt-5.5".to_string(),
6323                    file_backed_dynamic_context: true,
6324                    ..RuntimeConfig::default()
6325                },
6326            )
6327            .unwrap(),
6328        );
6329        let thread_id = runtime
6330            .create_thread(Some("Context failure continue".to_string()))
6331            .await
6332            .unwrap()
6333            .thread_id;
6334        let failed_turn = "failed-turn".to_string();
6335        runtime
6336            .persist_turn_item(
6337                &thread_id,
6338                &failed_turn,
6339                &TranscriptItem::UserMessage(UserMessage::text("old work ".repeat(10_000))),
6340            )
6341            .await
6342            .unwrap();
6343        runtime
6344            .persist_turn_item(
6345                &thread_id,
6346                &failed_turn,
6347                &TranscriptItem::Error(ErrorRecord {
6348                    message: "Your input exceeds the context window of this model. Please adjust your input and try again."
6349                        .to_string(),
6350                }),
6351            )
6352            .await
6353            .unwrap();
6354
6355        let mut events = runtime.subscribe_events();
6356        runtime
6357            .start_turn(StartTurnRequest {
6358                thread_id: thread_id.clone(),
6359                message: "continue".to_string(),
6360                images: Vec::new(),
6361                provider_override: None,
6362                model_override: None,
6363                reasoning_override: None,
6364                workspace: test_workspace(),
6365                instructions: InstructionBundle::default(),
6366                developer_context: None,
6367                task_ledger_required: false,
6368                service_tier_override: None,
6369            })
6370            .await
6371            .unwrap();
6372        loop {
6373            let envelope = tokio::time::timeout(std::time::Duration::from_secs(5), events.recv())
6374                .await
6375                .unwrap()
6376                .unwrap();
6377            if envelope.thread_id.as_deref() == Some(&thread_id)
6378                && matches!(envelope.event, RoderEvent::TurnCompleted(_))
6379            {
6380                break;
6381            }
6382        }
6383
6384        let request = captured.lock().unwrap().clone().unwrap();
6385        assert!(
6386            matches!(
6387                request.transcript.first(),
6388                Some(TranscriptItem::ContextCompaction(_))
6389            ),
6390            "provider request after context-window failure should start with local compaction"
6391        );
6392        assert!(
6393            request
6394                .transcript
6395                .iter()
6396                .any(|item| matches!(item, TranscriptItem::UserMessage(message) if message.text == "continue")),
6397            "current continue prompt must be preserved: {:?}",
6398            request.transcript
6399        );
6400        assert!(
6401            !request.transcript.iter().any(
6402                |item| matches!(item, TranscriptItem::Error(error) if error.message.contains("context window"))
6403            ),
6404            "raw prior context-window error should be summarized, not replayed: {:?}",
6405            request.transcript
6406        );
6407
6408        let _ = std::fs::remove_dir_all(thread_root);
6409    }
6410
6411    #[tokio::test]
6412    async fn workspace_for_thread_falls_back_when_metadata_is_missing() {
6413        let workspace = test_workspace();
6414        let mut builder = ExtensionRegistryBuilder::new();
6415        builder.inference_engine(Arc::new(FakeInferenceEngine));
6416        builder.thread_store_factory(Arc::new(MetadataMissingStoreFactory));
6417        let runtime = Runtime::new(
6418            builder.build().unwrap(),
6419            RuntimeConfig {
6420                workspace: Some(workspace.clone()),
6421                ..RuntimeConfig::default()
6422            },
6423        )
6424        .unwrap();
6425
6426        let resolved = runtime
6427            .workspace_for_thread(&ThreadId::from("thread-workflow"))
6428            .await
6429            .unwrap();
6430
6431        assert_eq!(resolved, workspace);
6432    }
6433
6434    #[tokio::test]
6435    async fn automations_can_create_project_thread_with_model_overrides() {
6436        let runtime = Runtime::fake().unwrap();
6437        let workspace = std::env::temp_dir().join("project");
6438        let metadata = runtime
6439            .create_thread_with(CreateThreadRequest {
6440                title: Some("Automation: nightly status".to_string()),
6441                workspace: workspace.display().to_string(),
6442                workspace_id: None,
6443                root_id: None,
6444                provider: Some("mock".to_string()),
6445                model: Some("mock".to_string()),
6446                selection_mode: None,
6447                tool_allowlist: Vec::new(),
6448                developer_instructions: None,
6449                external_tools: Vec::new(),
6450                runner: None,
6451            })
6452            .await
6453            .unwrap();
6454
6455        assert_eq!(
6456            metadata.title.as_deref(),
6457            Some("Automation: nightly status")
6458        );
6459        assert_eq!(metadata.workspace, workspace.display().to_string());
6460        assert_eq!(metadata.provider.as_deref(), Some("mock"));
6461        assert_eq!(metadata.model.as_deref(), Some("mock"));
6462    }
6463
6464    #[tokio::test]
6465    async fn prior_provider_compaction_boundary_is_used_on_next_turn() {
6466        let captured = Arc::new(StdMutex::new(None));
6467        let mut builder = ExtensionRegistryBuilder::new();
6468        builder.inference_engine(Arc::new(CapturingEngine {
6469            request: captured.clone(),
6470        }));
6471        let thread_root = std::env::temp_dir().join(format!(
6472            "roder-provider-compaction-boundary-{}",
6473            uuid::Uuid::new_v4()
6474        ));
6475        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
6476            base_path: thread_root.clone(),
6477        }));
6478        let runtime = Arc::new(
6479            Runtime::new(
6480                builder.build().unwrap(),
6481                RuntimeConfig {
6482                    default_provider: PROVIDER_MOCK.to_string(),
6483                    default_model: "gpt-5.5".to_string(),
6484                    ..RuntimeConfig::default()
6485                },
6486            )
6487            .unwrap(),
6488        );
6489        let thread_id = runtime
6490            .create_thread(Some("Provider compaction boundary".to_string()))
6491            .await
6492            .unwrap()
6493            .thread_id;
6494        let old_turn = "old-turn".to_string();
6495        runtime
6496            .persist_turn_item(
6497                &thread_id,
6498                &old_turn,
6499                &TranscriptItem::UserMessage(UserMessage::text(
6500                    "old history that must not be replayed after provider compaction",
6501                )),
6502            )
6503            .await
6504            .unwrap();
6505        runtime
6506            .persist_turn_item(
6507                &thread_id,
6508                &old_turn,
6509                &TranscriptItem::AssistantMessage(AssistantMessage {
6510                    text: "old answer".to_string(),
6511                    phase: None,
6512                }),
6513            )
6514            .await
6515            .unwrap();
6516        runtime
6517            .persist_turn_item(
6518                &thread_id,
6519                &old_turn,
6520                &TranscriptItem::ProviderMetadata(serde_json::json!({
6521                    "output": [{
6522                        "id": "cmp_1",
6523                        "type": "compaction",
6524                        "encrypted_content": "opaque-state"
6525                    }]
6526                })),
6527            )
6528            .await
6529            .unwrap();
6530        runtime
6531            .persist_turn_item(
6532                &thread_id,
6533                &old_turn,
6534                &TranscriptItem::AssistantMessage(AssistantMessage {
6535                    text: "after compact".to_string(),
6536                    phase: None,
6537                }),
6538            )
6539            .await
6540            .unwrap();
6541
6542        let mut events = runtime.subscribe_events();
6543        runtime
6544            .start_turn(StartTurnRequest {
6545                thread_id: thread_id.clone(),
6546                message: "continue".to_string(),
6547                images: Vec::new(),
6548                provider_override: None,
6549                model_override: None,
6550                reasoning_override: None,
6551                workspace: test_workspace(),
6552                instructions: InstructionBundle::default(),
6553                developer_context: None,
6554                task_ledger_required: false,
6555                service_tier_override: None,
6556            })
6557            .await
6558            .unwrap();
6559        loop {
6560            let envelope = tokio::time::timeout(std::time::Duration::from_secs(5), events.recv())
6561                .await
6562                .unwrap()
6563                .unwrap();
6564            if envelope.thread_id.as_deref() == Some(&thread_id)
6565                && matches!(envelope.event, RoderEvent::TurnCompleted(_))
6566            {
6567                break;
6568            }
6569        }
6570
6571        let request = captured.lock().unwrap().clone().unwrap();
6572        let compaction_idx = request.transcript.iter().position(|item| {
6573            matches!(
6574                item,
6575                TranscriptItem::ProviderMetadata(metadata)
6576                    if crate::compaction::provider_metadata_has_compaction(metadata)
6577            )
6578        });
6579        assert!(
6580            compaction_idx.is_some(),
6581            "next provider request should include the provider compaction item: {:?}",
6582            request.transcript
6583        );
6584        // Skill/context injectors may prepend non-history items, but no prior-turn
6585        // conversation may appear before the latest provider compaction boundary.
6586        let history_before_boundary =
6587            request
6588                .transcript
6589                .iter()
6590                .take(compaction_idx.unwrap())
6591                .any(|item| match item {
6592                    TranscriptItem::AssistantMessage(_)
6593                    | TranscriptItem::ToolCall(_)
6594                    | TranscriptItem::ToolResult(_) => true,
6595                    TranscriptItem::UserMessage(message) => {
6596                        !message.text.contains("<skills>") && message.text != "continue"
6597                    }
6598                    _ => false,
6599                });
6600        assert!(
6601            !history_before_boundary,
6602            "pre-provider-compaction conversation must not be replayed: {:?}",
6603            request.transcript
6604        );
6605        assert!(
6606            !request.transcript.iter().any(|item| {
6607                matches!(
6608                    item,
6609                    TranscriptItem::UserMessage(message)
6610                        if message.text.contains("old history that must not be replayed")
6611                )
6612            }),
6613            "pre-provider-compaction history must not be replayed: {:?}",
6614            request.transcript
6615        );
6616        assert!(
6617            request.transcript.iter().any(|item| {
6618                matches!(
6619                    item,
6620                    TranscriptItem::UserMessage(message) if message.text == "continue"
6621                )
6622            }),
6623            "current continue prompt must be preserved: {:?}",
6624            request.transcript
6625        );
6626        assert_eq!(
6627            request.runtime.auto_compact_token_limit,
6628            Some(945_000),
6629            "gpt-5.5 should keep server-side auto compaction configured"
6630        );
6631
6632        let _ = std::fs::remove_dir_all(thread_root);
6633    }
6634
6635    #[test]
6636    fn server_side_compaction_is_only_enabled_for_supported_models() {
6637        let cfg = RuntimeConfig {
6638            auto_compact_token_limit: Some(123_456),
6639            ..RuntimeConfig::default()
6640        };
6641
6642        assert_eq!(server_side_compaction_threshold(&cfg, "mock"), None);
6643        assert_eq!(
6644            server_side_compaction_threshold(&cfg, "codex-auto-review"),
6645            None
6646        );
6647    }
6648
6649    #[test]
6650    fn reasoning_is_disabled_for_models_without_reasoning_support() {
6651        let cfg = RuntimeConfig {
6652            reasoning: Some(REASONING_HIGH.to_string()),
6653            ..RuntimeConfig::default()
6654        };
6655
6656        assert_eq!(
6657            effective_reasoning_for_model(&cfg, "claude-haiku-4-5-20251001"),
6658            REASONING_NONE
6659        );
6660        assert_eq!(
6661            reasoning_for_model(&cfg, "claude-haiku-4-5-20251001"),
6662            ReasoningConfig::default()
6663        );
6664    }
6665
6666    #[test]
6667    fn unsupported_configured_reasoning_falls_back_to_model_default() {
6668        let cfg = RuntimeConfig {
6669            reasoning: Some(REASONING_MINIMAL.to_string()),
6670            ..RuntimeConfig::default()
6671        };
6672
6673        assert_eq!(
6674            effective_reasoning_for_model(&cfg, "gpt-5.5"),
6675            REASONING_MEDIUM
6676        );
6677    }
6678
6679    #[test]
6680    fn unsupported_configured_gemini_reasoning_is_rejected() {
6681        let mut builder = ExtensionRegistryBuilder::new();
6682        builder.inference_engine(std::sync::Arc::new(FakeInferenceEngine));
6683
6684        let err = match Runtime::new(
6685            builder.build().unwrap(),
6686            RuntimeConfig {
6687                default_model: "gemini-3.5-flash".to_string(),
6688                reasoning: Some(REASONING_XHIGH.to_string()),
6689                ..RuntimeConfig::default()
6690            },
6691        ) {
6692            Ok(_) => panic!("expected unsupported Gemini reasoning to be rejected"),
6693            Err(err) => err,
6694        };
6695
6696        assert!(
6697            err.to_string()
6698                .contains("model gemini-3.5-flash does not support reasoning effort xhigh")
6699        );
6700    }
6701
6702    #[tokio::test]
6703    async fn selecting_none_for_non_reasoning_model_preserves_stored_preference() {
6704        let runtime = Runtime::new(
6705            Runtime::fake().unwrap().registry,
6706            RuntimeConfig {
6707                reasoning: Some(REASONING_HIGH.to_string()),
6708                ..RuntimeConfig::default()
6709            },
6710        )
6711        .unwrap();
6712
6713        let cfg = runtime
6714            .select_provider(
6715                roder_api::catalog::PROVIDER_MOCK.to_string(),
6716                Some("claude-haiku-4-5-20251001".to_string()),
6717                Some(REASONING_NONE.to_string()),
6718            )
6719            .await
6720            .unwrap();
6721
6722        assert_eq!(cfg.reasoning.as_deref(), Some(REASONING_HIGH));
6723        assert_eq!(runtime.effective_reasoning().await, REASONING_NONE);
6724    }
6725
6726    #[tokio::test]
6727    async fn selecting_none_for_model_that_supports_none_updates_preference() {
6728        let runtime = Runtime::new(
6729            Runtime::fake().unwrap().registry,
6730            RuntimeConfig {
6731                reasoning: Some(REASONING_HIGH.to_string()),
6732                ..RuntimeConfig::default()
6733            },
6734        )
6735        .unwrap();
6736
6737        let cfg = runtime
6738            .select_provider(
6739                roder_api::catalog::PROVIDER_MOCK.to_string(),
6740                Some("mock".to_string()),
6741                Some(REASONING_NONE.to_string()),
6742            )
6743            .await
6744            .unwrap();
6745
6746        assert_eq!(cfg.reasoning.as_deref(), Some(REASONING_NONE));
6747    }
6748
6749    #[test]
6750    fn parallel_tool_calls_default_on_with_model_override() {
6751        assert!(parallel_tool_calls_for_model(
6752            &RuntimeConfig::default(),
6753            "custom-model"
6754        ));
6755
6756        let cfg = RuntimeConfig {
6757            model_parallel_tool_calls: std::collections::HashMap::from([(
6758                "custom-model".to_string(),
6759                false,
6760            )]),
6761            ..RuntimeConfig::default()
6762        };
6763
6764        assert!(!parallel_tool_calls_for_model(&cfg, "custom-model"));
6765        assert!(parallel_tool_calls_for_model(&cfg, "other-model"));
6766    }
6767
6768    #[test]
6769    fn profile_parallel_tool_calls_applies_between_config_and_default() {
6770        let cfg = RuntimeConfig {
6771            model_profiles: std::collections::HashMap::from([(
6772                "gpt-5.5".to_string(),
6773                test_model_profile("gpt-5.5"),
6774            )]),
6775            ..RuntimeConfig::default()
6776        };
6777
6778        assert!(!parallel_tool_calls_for_model(&cfg, "gpt-5.5"));
6779
6780        let cfg = RuntimeConfig {
6781            model_parallel_tool_calls: std::collections::HashMap::from([(
6782                "gpt-5.5".to_string(),
6783                true,
6784            )]),
6785            ..cfg
6786        };
6787
6788        assert!(parallel_tool_calls_for_model(&cfg, "gpt-5.5"));
6789    }
6790
6791    struct CapturingEngine {
6792        request: Arc<StdMutex<Option<AgentInferenceRequest>>>,
6793    }
6794
6795    #[async_trait::async_trait]
6796    impl InferenceEngine for CapturingEngine {
6797        fn id(&self) -> String {
6798            roder_api::catalog::PROVIDER_MOCK.to_string()
6799        }
6800
6801        fn capabilities(&self) -> InferenceCapabilities {
6802            InferenceCapabilities::coding_agent_default()
6803        }
6804
6805        async fn list_models(
6806            &self,
6807            _ctx: InferenceProviderContext<'_>,
6808        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
6809            Ok(roder_api::catalog::models_for_provider(
6810                roder_api::catalog::PROVIDER_MOCK,
6811                true,
6812            ))
6813        }
6814
6815        async fn stream_turn(
6816            &self,
6817            _ctx: InferenceTurnContext<'_>,
6818            request: AgentInferenceRequest,
6819        ) -> anyhow::Result<InferenceEventStream> {
6820            *self.request.lock().unwrap() = Some(request);
6821            Ok(Box::pin(stream::iter(vec![
6822                Ok(InferenceEvent::MessageDelta(MessageDelta {
6823                    text: "done".to_string(),
6824                    phase: None,
6825                })),
6826                Ok(InferenceEvent::Completed(CompletionMetadata {
6827                    stop_reason: Some("stop".to_string()),
6828                    provider_response_id: None,
6829                })),
6830            ])))
6831        }
6832    }
6833
6834    struct RoutingCaptureEngine {
6835        id: &'static str,
6836        models: Vec<ModelDescriptor>,
6837        requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
6838    }
6839
6840    #[async_trait::async_trait]
6841    impl InferenceEngine for RoutingCaptureEngine {
6842        fn id(&self) -> String {
6843            self.id.to_string()
6844        }
6845
6846        fn capabilities(&self) -> InferenceCapabilities {
6847            InferenceCapabilities::coding_agent_default()
6848        }
6849
6850        async fn list_models(
6851            &self,
6852            _ctx: InferenceProviderContext<'_>,
6853        ) -> anyhow::Result<Vec<ModelDescriptor>> {
6854            Ok(self.models.clone())
6855        }
6856
6857        async fn stream_turn(
6858            &self,
6859            _ctx: InferenceTurnContext<'_>,
6860            request: AgentInferenceRequest,
6861        ) -> anyhow::Result<InferenceEventStream> {
6862            self.requests.lock().unwrap().push(request);
6863            Ok(Box::pin(stream::iter(vec![
6864                Ok(InferenceEvent::MessageDelta(MessageDelta {
6865                    text: "routed".to_string(),
6866                    phase: None,
6867                })),
6868                Ok(InferenceEvent::Completed(CompletionMetadata {
6869                    stop_reason: Some("stop".to_string()),
6870                    provider_response_id: None,
6871                })),
6872            ])))
6873        }
6874    }
6875
6876    struct StaticRouter {
6877        id: &'static str,
6878        decision: InferenceRoutingDecision,
6879        contexts: Arc<StdMutex<Vec<InferenceRoutingContext>>>,
6880    }
6881
6882    #[async_trait::async_trait]
6883    impl InferenceRouter for StaticRouter {
6884        fn id(&self) -> String {
6885            self.id.to_string()
6886        }
6887
6888        async fn route(
6889            &self,
6890            context: InferenceRoutingContext,
6891        ) -> anyhow::Result<InferenceRoutingDecision> {
6892            self.contexts.lock().unwrap().push(context);
6893            Ok(self.decision.clone())
6894        }
6895    }
6896
6897    fn routing_test_model(id: &str, supported_reasoning: &[&str]) -> ModelDescriptor {
6898        ModelDescriptor {
6899            id: id.to_string(),
6900            name: id.to_string(),
6901            context_window: Some(128_000),
6902            default_reasoning: supported_reasoning
6903                .first()
6904                .map(|effort| (*effort).to_string()),
6905            supported_reasoning: supported_reasoning
6906                .iter()
6907                .map(|effort| ReasoningEffortDescriptor {
6908                    effort: (*effort).to_string(),
6909                    description: format!("{effort} reasoning"),
6910                })
6911                .collect(),
6912        }
6913    }
6914
6915    struct TaskLedgerCompletionGateEngine {
6916        calls: StdMutex<u32>,
6917        requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
6918    }
6919
6920    #[async_trait::async_trait]
6921    impl InferenceEngine for TaskLedgerCompletionGateEngine {
6922        fn id(&self) -> String {
6923            roder_api::catalog::PROVIDER_MOCK.to_string()
6924        }
6925
6926        fn capabilities(&self) -> InferenceCapabilities {
6927            InferenceCapabilities::coding_agent_default()
6928        }
6929
6930        async fn list_models(
6931            &self,
6932            _ctx: InferenceProviderContext<'_>,
6933        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
6934            Ok(roder_api::catalog::models_for_provider(
6935                roder_api::catalog::PROVIDER_MOCK,
6936                true,
6937            ))
6938        }
6939
6940        async fn stream_turn(
6941            &self,
6942            _ctx: InferenceTurnContext<'_>,
6943            request: AgentInferenceRequest,
6944        ) -> anyhow::Result<InferenceEventStream> {
6945            self.requests.lock().unwrap().push(request);
6946            let mut calls = self.calls.lock().unwrap();
6947            *calls += 1;
6948            let events = match *calls {
6949                1 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
6950                    id: "ledger-open".to_string(),
6951                    name: TASK_LEDGER_TOOL_NAME.to_string(),
6952                    arguments: serde_json::json!({
6953                        "tasks": [
6954                            {
6955                                "id": "inspect",
6956                                "content": "Inspect local assets",
6957                                "status": "completed",
6958                                "evidence": "listed workspace"
6959                            },
6960                            {
6961                                "id": "write",
6962                                "content": "Write /app/result.txt",
6963                                "status": "pending"
6964                            }
6965                        ],
6966                        "requireCompletionEvidence": true
6967                    })
6968                    .to_string(),
6969                }))],
6970                3 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
6971                    id: "ledger-complete".to_string(),
6972                    name: TASK_LEDGER_TOOL_NAME.to_string(),
6973                    arguments: serde_json::json!({
6974                        "tasks": [
6975                            {
6976                                "id": "inspect",
6977                                "content": "Inspect local assets",
6978                                "status": "completed",
6979                                "evidence": "listed workspace"
6980                            },
6981                            {
6982                                "id": "write",
6983                                "content": "Write /app/result.txt",
6984                                "status": "completed",
6985                                "evidence": "wrote answer"
6986                            }
6987                        ],
6988                        "requireCompletionEvidence": true
6989                    })
6990                    .to_string(),
6991                }))],
6992                _ => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
6993                    text: "final".to_string(),
6994                    phase: None,
6995                }))],
6996            };
6997            Ok(Box::pin(stream::iter(events.into_iter().chain(
6998                std::iter::once(Ok(InferenceEvent::Completed(CompletionMetadata {
6999                    stop_reason: Some("stop".to_string()),
7000                    provider_response_id: None,
7001                }))),
7002            ))))
7003        }
7004    }
7005
7006    struct VerificationGateEngine {
7007        calls: StdMutex<u32>,
7008    }
7009
7010    #[async_trait::async_trait]
7011    impl InferenceEngine for VerificationGateEngine {
7012        fn id(&self) -> String {
7013            roder_api::catalog::PROVIDER_MOCK.to_string()
7014        }
7015
7016        fn capabilities(&self) -> InferenceCapabilities {
7017            InferenceCapabilities::coding_agent_default()
7018        }
7019
7020        async fn list_models(
7021            &self,
7022            _ctx: InferenceProviderContext<'_>,
7023        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7024            Ok(roder_api::catalog::models_for_provider(
7025                roder_api::catalog::PROVIDER_MOCK,
7026                true,
7027            ))
7028        }
7029
7030        async fn stream_turn(
7031            &self,
7032            _ctx: InferenceTurnContext<'_>,
7033            request: AgentInferenceRequest,
7034        ) -> anyhow::Result<InferenceEventStream> {
7035            let mut calls = self.calls.lock().unwrap();
7036            *calls += 1;
7037            let events = match *calls {
7038                1 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7039                    id: "write-1".to_string(),
7040                    name: "write_file".to_string(),
7041                    arguments: serde_json::json!({
7042                        "path": "src/lib.rs",
7043                        "content": "pub fn answer() -> u8 { 42 }\n"
7044                    })
7045                    .to_string(),
7046                }))],
7047                2 => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
7048                    text: "done too early".to_string(),
7049                    phase: None,
7050                }))],
7051                3 if request.transcript.iter().any(|item| {
7052                    matches!(
7053                        item,
7054                        TranscriptItem::UserMessage(message)
7055                            if message.text.contains("Verification gate blocked final completion")
7056                    )
7057                }) =>
7058                {
7059                    vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7060                        id: "verify-1".to_string(),
7061                        name: crate::verification_gate::VERIFICATION_TOOL_NAME.to_string(),
7062                        arguments: serde_json::json!({
7063                            "originalTask": "write code",
7064                            "changedFiles": ["src/lib.rs"],
7065                            "toolEvidence": ["write_file wrote src/lib.rs"],
7066                            "testsRun": ["cargo test -p roder-core verification_gate"],
7067                            "openGaps": [],
7068                            "status": "completed"
7069                        })
7070                        .to_string(),
7071                    }))]
7072                }
7073                _ => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
7074                    text: "verified final".to_string(),
7075                    phase: None,
7076                }))],
7077            };
7078            Ok(Box::pin(stream::iter(events.into_iter().chain(
7079                std::iter::once(Ok(InferenceEvent::Completed(CompletionMetadata {
7080                    stop_reason: Some("stop".to_string()),
7081                    provider_response_id: None,
7082                }))),
7083            ))))
7084        }
7085    }
7086
7087    struct SpeedPolicyEngine {
7088        calls: StdMutex<u32>,
7089        requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
7090    }
7091
7092    #[async_trait::async_trait]
7093    impl InferenceEngine for SpeedPolicyEngine {
7094        fn id(&self) -> String {
7095            roder_api::catalog::PROVIDER_MOCK.to_string()
7096        }
7097
7098        fn capabilities(&self) -> InferenceCapabilities {
7099            InferenceCapabilities::coding_agent_default()
7100        }
7101
7102        async fn list_models(
7103            &self,
7104            _ctx: InferenceProviderContext<'_>,
7105        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7106            Ok(roder_api::catalog::models_for_provider(
7107                roder_api::catalog::PROVIDER_MOCK,
7108                true,
7109            ))
7110        }
7111
7112        async fn stream_turn(
7113            &self,
7114            _ctx: InferenceTurnContext<'_>,
7115            request: AgentInferenceRequest,
7116        ) -> anyhow::Result<InferenceEventStream> {
7117            self.requests.lock().unwrap().push(request.clone());
7118            let mut calls = self.calls.lock().unwrap();
7119            *calls += 1;
7120            let events = match *calls {
7121                1 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7122                    id: "write-1".to_string(),
7123                    name: "write_file".to_string(),
7124                    arguments: serde_json::json!({
7125                        "path": "src/lib.rs",
7126                        "content": "pub fn answer() -> u8 { 42 }\n"
7127                    })
7128                    .to_string(),
7129                }))],
7130                3 if request.transcript.iter().any(|item| {
7131                    matches!(
7132                        item,
7133                        TranscriptItem::UserMessage(message)
7134                            if message.text.contains("Verification gate blocked final completion")
7135                    )
7136                }) =>
7137                {
7138                    vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7139                        id: "verify-1".to_string(),
7140                        name: crate::verification_gate::VERIFICATION_TOOL_NAME.to_string(),
7141                        arguments: serde_json::json!({
7142                            "originalTask": "write code",
7143                            "changedFiles": ["src/lib.rs"],
7144                            "toolEvidence": ["write_file wrote src/lib.rs"],
7145                            "testsRun": ["cargo test -p roder-core speed_policy"],
7146                            "openGaps": [],
7147                            "status": "completed"
7148                        })
7149                        .to_string(),
7150                    }))]
7151                }
7152                _ => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
7153                    text: "done".to_string(),
7154                    phase: None,
7155                }))],
7156            };
7157            Ok(Box::pin(stream::iter(events.into_iter().chain(
7158                std::iter::once(Ok(InferenceEvent::Completed(CompletionMetadata {
7159                    stop_reason: Some("stop".to_string()),
7160                    provider_response_id: None,
7161                }))),
7162            ))))
7163        }
7164    }
7165
7166    struct SwitchCaptureEngine {
7167        requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
7168    }
7169
7170    #[async_trait::async_trait]
7171    impl InferenceEngine for SwitchCaptureEngine {
7172        fn id(&self) -> String {
7173            roder_api::catalog::PROVIDER_MOCK.to_string()
7174        }
7175
7176        fn capabilities(&self) -> InferenceCapabilities {
7177            InferenceCapabilities::coding_agent_default()
7178        }
7179
7180        async fn list_models(
7181            &self,
7182            _ctx: InferenceProviderContext<'_>,
7183        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7184            Ok(roder_api::catalog::models_for_provider(
7185                roder_api::catalog::PROVIDER_MOCK,
7186                true,
7187            ))
7188        }
7189
7190        async fn stream_turn(
7191            &self,
7192            _ctx: InferenceTurnContext<'_>,
7193            request: AgentInferenceRequest,
7194        ) -> anyhow::Result<InferenceEventStream> {
7195            self.requests.lock().unwrap().push(request);
7196            Ok(Box::pin(stream::iter(vec![
7197                Ok(InferenceEvent::MessageDelta(MessageDelta {
7198                    text: "done".to_string(),
7199                    phase: None,
7200                })),
7201                Ok(InferenceEvent::Completed(CompletionMetadata {
7202                    stop_reason: Some("stop".to_string()),
7203                    provider_response_id: None,
7204                })),
7205            ])))
7206        }
7207    }
7208
7209    struct DeadlineEngine;
7210
7211    #[async_trait::async_trait]
7212    impl InferenceEngine for DeadlineEngine {
7213        fn id(&self) -> String {
7214            roder_api::catalog::PROVIDER_MOCK.to_string()
7215        }
7216
7217        fn capabilities(&self) -> InferenceCapabilities {
7218            InferenceCapabilities::coding_agent_default()
7219        }
7220
7221        async fn list_models(
7222            &self,
7223            _ctx: InferenceProviderContext<'_>,
7224        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7225            Ok(Vec::new())
7226        }
7227
7228        async fn stream_turn(
7229            &self,
7230            _ctx: InferenceTurnContext<'_>,
7231            _request: AgentInferenceRequest,
7232        ) -> anyhow::Result<InferenceEventStream> {
7233            Ok(Box::pin(stream::once(async {
7234                tokio::time::sleep(std::time::Duration::from_secs(60)).await;
7235                Ok(InferenceEvent::MessageDelta(MessageDelta {
7236                    text: "too late".to_string(),
7237                    phase: None,
7238                }))
7239            })))
7240        }
7241    }
7242
7243    struct WriteFileContributor;
7244
7245    impl ToolContributor for WriteFileContributor {
7246        fn id(&self) -> String {
7247            "test-write".to_string()
7248        }
7249
7250        fn contribute(&self, registry: &mut ToolRegistry) -> anyhow::Result<()> {
7251            registry.register(Arc::new(WriteFileTool))
7252        }
7253    }
7254
7255    struct WriteFileTool;
7256
7257    #[async_trait::async_trait]
7258    impl ToolExecutor for WriteFileTool {
7259        fn spec(&self) -> ToolSpec {
7260            ToolSpec {
7261                name: "write_file".to_string(),
7262                description: "Write a test file.".to_string(),
7263                parameters: serde_json::json!({
7264                    "type": "object",
7265                    "properties": {
7266                        "path": { "type": "string" },
7267                        "content": { "type": "string" }
7268                    },
7269                    "required": ["path", "content"],
7270                    "additionalProperties": false
7271                }),
7272            }
7273        }
7274
7275        async fn execute(
7276            &self,
7277            _ctx: ToolExecutionContext,
7278            call: ToolCall,
7279        ) -> anyhow::Result<ToolResult> {
7280            let path = call
7281                .arguments
7282                .get("path")
7283                .and_then(serde_json::Value::as_str)
7284                .unwrap_or("src/lib.rs");
7285            Ok(ToolResult {
7286                id: call.id,
7287                name: call.name,
7288                text: format!("wrote {path}"),
7289                data: serde_json::json!({ "path": path }),
7290                is_error: false,
7291            })
7292        }
7293    }
7294
7295    struct ProfileToolContributor;
7296
7297    impl ToolContributor for ProfileToolContributor {
7298        fn id(&self) -> String {
7299            "profile-tools".to_string()
7300        }
7301
7302        fn contribute(&self, registry: &mut ToolRegistry) -> anyhow::Result<()> {
7303            for name in ["apply_patch", "edit", "multi_edit", "write_file"] {
7304                registry.register(Arc::new(ProfileTool {
7305                    name: name.to_string(),
7306                }))?;
7307            }
7308            Ok(())
7309        }
7310    }
7311
7312    struct ProfileTool {
7313        name: String,
7314    }
7315
7316    #[async_trait::async_trait]
7317    impl ToolExecutor for ProfileTool {
7318        fn spec(&self) -> ToolSpec {
7319            ToolSpec {
7320                name: self.name.clone(),
7321                description: format!("{} test tool", self.name),
7322                parameters: serde_json::json!({
7323                    "type": "object",
7324                    "properties": {
7325                        "path": { "type": "string" },
7326                        "content": { "type": "string" }
7327                    },
7328                    "required": ["path", "content"],
7329                    "additionalProperties": false
7330                }),
7331            }
7332        }
7333
7334        async fn execute(
7335            &self,
7336            _ctx: ToolExecutionContext,
7337            call: ToolCall,
7338        ) -> anyhow::Result<ToolResult> {
7339            Ok(ToolResult {
7340                id: call.id,
7341                name: call.name,
7342                text: "ok".to_string(),
7343                data: serde_json::json!({}),
7344                is_error: false,
7345            })
7346        }
7347    }
7348
7349    fn test_model_profile(model: &str) -> ModelHarnessProfile {
7350        ModelHarnessProfile {
7351            model: model.to_string(),
7352            provider: roder_api::catalog::PROVIDER_OPENAI.to_string(),
7353            provider_family: ProviderFamily::OpenAi,
7354            edit_tool: Some(EDIT_TOOL_EDIT.to_string()),
7355            schema_policy: ModelSchemaPolicy::StandardRequiredFirst,
7356            instruction_overlay: ModelInstructionOverlay::IntuitiveContext,
7357            reasoning: ModelProfileReasoning {
7358                orientation: Some(REASONING_LOW.to_string()),
7359                execution: Some(REASONING_LOW.to_string()),
7360                verification: Some(REASONING_LOW.to_string()),
7361                recovery: Some(REASONING_LOW.to_string()),
7362            },
7363            parallel_tool_calls: Some(false),
7364            auto_compact_token_limit: Some(123_000),
7365        }
7366    }
7367
7368    async fn captured_profile_request(cfg: RuntimeConfig) -> AgentInferenceRequest {
7369        let captured = Arc::new(StdMutex::new(None));
7370        let mut builder = ExtensionRegistryBuilder::new();
7371        builder.inference_engine(Arc::new(CapturingEngine {
7372            request: captured.clone(),
7373        }));
7374        builder.tool_contributor(Arc::new(ProfileToolContributor));
7375        let runtime = Arc::new(Runtime::new(builder.build().unwrap(), cfg).unwrap());
7376        let mut rx = runtime.subscribe_events();
7377        let turn_id = runtime
7378            .start_turn(StartTurnRequest {
7379                thread_id: "thread-model-profile".to_string(),
7380                message: "use profile knobs".to_string(),
7381                images: Vec::new(),
7382                provider_override: None,
7383                model_override: None,
7384                reasoning_override: None,
7385                workspace: test_workspace(),
7386                instructions: InstructionBundle {
7387                    system: None,
7388                    developer: Some("base developer".to_string()),
7389                    developer_context: None,
7390                },
7391                developer_context: None,
7392                task_ledger_required: false,
7393                service_tier_override: None,
7394            })
7395            .await
7396            .unwrap();
7397
7398        tokio::time::timeout(std::time::Duration::from_secs(5), async {
7399            loop {
7400                let envelope = rx.recv().await.unwrap();
7401                if envelope.turn_id.as_deref() != Some(&turn_id) {
7402                    continue;
7403                }
7404                match envelope.event {
7405                    RoderEvent::TurnCompleted(_) => break,
7406                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7407                    _ => {}
7408                }
7409            }
7410        })
7411        .await
7412        .unwrap();
7413
7414        captured.lock().unwrap().clone().unwrap()
7415    }
7416
7417    struct ToolThenStopEngine {
7418        calls: StdMutex<u32>,
7419    }
7420
7421    #[async_trait::async_trait]
7422    impl InferenceEngine for ToolThenStopEngine {
7423        fn id(&self) -> String {
7424            roder_api::catalog::PROVIDER_MOCK.to_string()
7425        }
7426
7427        fn capabilities(&self) -> InferenceCapabilities {
7428            InferenceCapabilities::coding_agent_default()
7429        }
7430
7431        async fn list_models(
7432            &self,
7433            _ctx: InferenceProviderContext<'_>,
7434        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7435            Ok(roder_api::catalog::models_for_provider(
7436                roder_api::catalog::PROVIDER_MOCK,
7437                true,
7438            ))
7439        }
7440
7441        async fn stream_turn(
7442            &self,
7443            _ctx: InferenceTurnContext<'_>,
7444            _request: AgentInferenceRequest,
7445        ) -> anyhow::Result<InferenceEventStream> {
7446            let mut calls = self.calls.lock().unwrap();
7447            *calls += 1;
7448            let events = match *calls {
7449                1 => vec![
7450                    Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7451                        id: "write-1".to_string(),
7452                        name: "write_file".to_string(),
7453                        arguments: serde_json::json!({
7454                            "path": "src/lib.rs",
7455                            "content": "pub fn answer() -> u8 { 42 }\n"
7456                        })
7457                        .to_string(),
7458                    })),
7459                    Ok(InferenceEvent::Completed(CompletionMetadata {
7460                        stop_reason: Some("tool_use".to_string()),
7461                        provider_response_id: None,
7462                    })),
7463                ],
7464                _ => vec![
7465                    Ok(InferenceEvent::MessageDelta(MessageDelta {
7466                        text: "final".to_string(),
7467                        phase: None,
7468                    })),
7469                    Ok(InferenceEvent::Completed(CompletionMetadata {
7470                        stop_reason: Some("end_turn".to_string()),
7471                        provider_response_id: None,
7472                    })),
7473                ],
7474            };
7475            Ok(Box::pin(stream::iter(events)))
7476        }
7477    }
7478
7479    #[tokio::test]
7480    async fn turn_completed_reports_terminal_step_finish_reason() {
7481        let mut builder = ExtensionRegistryBuilder::new();
7482        builder.inference_engine(Arc::new(ToolThenStopEngine {
7483            calls: StdMutex::new(0),
7484        }));
7485        builder.tool_contributor(Arc::new(WriteFileContributor));
7486        let runtime = Arc::new(
7487            Runtime::new(
7488                builder.build().unwrap(),
7489                RuntimeConfig {
7490                    policy_mode: PolicyMode::Bypass,
7491                    agent_swarm_mode: false,
7492                    ultra_mode: false,
7493                    ..RuntimeConfig::default()
7494                },
7495            )
7496            .unwrap(),
7497        );
7498        let mut rx = runtime.subscribe_events();
7499        let turn_id = runtime
7500            .start_turn(StartTurnRequest {
7501                thread_id: "thread-finish-reason".to_string(),
7502                message: "write then finish".to_string(),
7503                images: Vec::new(),
7504                provider_override: None,
7505                model_override: None,
7506                reasoning_override: None,
7507                workspace: test_workspace(),
7508                instructions: InstructionBundle {
7509                    system: None,
7510                    developer: None,
7511                    developer_context: None,
7512                },
7513                developer_context: None,
7514                task_ledger_required: false,
7515                service_tier_override: None,
7516            })
7517            .await
7518            .unwrap();
7519
7520        let completed = tokio::time::timeout(std::time::Duration::from_secs(5), async {
7521            loop {
7522                let envelope = rx.recv().await.unwrap();
7523                if envelope.turn_id.as_deref() != Some(&turn_id) {
7524                    continue;
7525                }
7526                match envelope.event {
7527                    RoderEvent::TurnCompleted(event) => break event,
7528                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7529                    _ => {}
7530                }
7531            }
7532        })
7533        .await
7534        .unwrap();
7535
7536        // The mid-turn tool_use stop reason must not leak; the terminal
7537        // end_turn step decides the turn's finish reason.
7538        assert_eq!(completed.finish_reason.as_deref(), Some("stop"));
7539    }
7540
7541    #[tokio::test]
7542    async fn inference_router_selection_changes_request_model_and_records_event() {
7543        let default_requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7544        let routed_requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7545        let contexts = Arc::new(StdMutex::new(Vec::<InferenceRoutingContext>::new()));
7546        let selected = ModelSelection {
7547            provider: "routed-provider".to_string(),
7548            model: "routed-model".to_string(),
7549        };
7550        let default = ModelSelection {
7551            provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7552            model: "mock".to_string(),
7553        };
7554        let decision = InferenceRoutingDecision {
7555            reasoning: Some(ReasoningConfig {
7556                enabled: true,
7557                level: Some(REASONING_LOW.to_string()),
7558            }),
7559            confidence: Some(0.91),
7560            baseline: Some(default.clone()),
7561            matched_signals: vec![roder_api::inference_routing::InferenceRoutingSignal::new(
7562                "intent", "routine",
7563            )],
7564            ..InferenceRoutingDecision::selected("test-router", selected.clone(), "routine request")
7565        };
7566
7567        let mut builder = ExtensionRegistryBuilder::new();
7568        builder.inference_engine(Arc::new(RoutingCaptureEngine {
7569            id: roder_api::catalog::PROVIDER_MOCK,
7570            models: vec![routing_test_model("mock", &[REASONING_LOW])],
7571            requests: default_requests.clone(),
7572        }));
7573        builder.inference_engine(Arc::new(RoutingCaptureEngine {
7574            id: "routed-provider",
7575            models: vec![routing_test_model(
7576                "routed-model",
7577                &[REASONING_LOW, REASONING_MEDIUM],
7578            )],
7579            requests: routed_requests.clone(),
7580        }));
7581        builder.inference_router(Arc::new(StaticRouter {
7582            id: "test-router",
7583            decision,
7584            contexts: contexts.clone(),
7585        }));
7586        let thread_root =
7587            std::env::temp_dir().join(format!("roder-routing-auto-{}", uuid::Uuid::new_v4()));
7588        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
7589            base_path: thread_root.clone(),
7590        }));
7591        let runtime = Arc::new(
7592            Runtime::new(
7593                builder.build().unwrap(),
7594                RuntimeConfig {
7595                    default_provider: default.provider.clone(),
7596                    default_model: default.model.clone(),
7597                    ..RuntimeConfig::default()
7598                },
7599            )
7600            .unwrap(),
7601        );
7602        let thread_id = runtime
7603            .create_thread_with(CreateThreadRequest {
7604                title: Some("Routing auto".to_string()),
7605                workspace: test_workspace(),
7606                workspace_id: None,
7607                root_id: None,
7608                provider: Some(default.provider.clone()),
7609                model: Some(default.model.clone()),
7610                tool_allowlist: Vec::new(),
7611                developer_instructions: None,
7612                external_tools: Vec::new(),
7613                selection_mode: Some(ModelSelectionMode::auto(
7614                    "test-router:coding",
7615                    "test-router",
7616                    "Auto: Coding",
7617                    default.clone(),
7618                    Some("coding".to_string()),
7619                    None,
7620                )),
7621                runner: None,
7622            })
7623            .await
7624            .unwrap()
7625            .thread_id;
7626        let mut rx = runtime.subscribe_events();
7627        let turn_id = runtime
7628            .start_turn(StartTurnRequest {
7629                thread_id: thread_id.clone(),
7630                message: "small cleanup".to_string(),
7631                images: Vec::new(),
7632                provider_override: None,
7633                model_override: None,
7634                reasoning_override: None,
7635                workspace: test_workspace(),
7636                instructions: InstructionBundle::default(),
7637                developer_context: None,
7638                task_ledger_required: false,
7639                service_tier_override: None,
7640            })
7641            .await
7642            .unwrap();
7643
7644        let mut routing_event = None;
7645        let mut inference_started = None;
7646        tokio::time::timeout(std::time::Duration::from_secs(5), async {
7647            loop {
7648                let envelope = rx.recv().await.unwrap();
7649                if envelope.turn_id.as_deref() != Some(&turn_id) {
7650                    continue;
7651                }
7652                match envelope.event {
7653                    RoderEvent::InferenceRoutingDecision(event) => {
7654                        routing_event = Some(event);
7655                    }
7656                    RoderEvent::InferenceStarted(event) => {
7657                        inference_started = Some(event);
7658                    }
7659                    RoderEvent::TurnCompleted(_) => break,
7660                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7661                    _ => {}
7662                }
7663            }
7664        })
7665        .await
7666        .unwrap();
7667
7668        assert!(default_requests.lock().unwrap().is_empty());
7669        let routed_requests = routed_requests.lock().unwrap();
7670        assert_eq!(routed_requests.len(), 1);
7671        assert_eq!(routed_requests[0].model, selected);
7672        assert_eq!(
7673            routed_requests[0].reasoning.level.as_deref(),
7674            Some(REASONING_LOW)
7675        );
7676        assert_eq!(
7677            routed_requests[0].metadata["inferenceRouting"]["outcome"],
7678            "selected"
7679        );
7680
7681        let routing_event = routing_event.expect("routing decision event");
7682        assert_eq!(routing_event.default_selection, default);
7683        assert_eq!(routing_event.selected_selection, selected);
7684        assert_eq!(
7685            routing_event.decision.outcome,
7686            InferenceRoutingOutcome::Selected
7687        );
7688        assert_eq!(
7689            inference_started.expect("inference started event").model,
7690            selected
7691        );
7692
7693        let contexts = contexts.lock().unwrap();
7694        assert_eq!(contexts.len(), 1);
7695        assert_eq!(contexts[0].default_selection, default);
7696        assert_eq!(contexts[0].candidates.len(), 2);
7697        assert!(
7698            contexts[0]
7699                .signals
7700                .iter()
7701                .any(|signal| signal.key == "profile" && signal.value == "coding")
7702        );
7703        let _ = std::fs::remove_dir_all(thread_root);
7704    }
7705
7706    #[tokio::test]
7707    async fn inference_router_is_bypassed_for_explicit_selection() {
7708        let requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7709        let contexts = Arc::new(StdMutex::new(Vec::<InferenceRoutingContext>::new()));
7710        let selected = ModelSelection {
7711            provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7712            model: "mock".to_string(),
7713        };
7714
7715        let mut builder = ExtensionRegistryBuilder::new();
7716        builder.inference_engine(Arc::new(RoutingCaptureEngine {
7717            id: roder_api::catalog::PROVIDER_MOCK,
7718            models: vec![routing_test_model("mock", &[REASONING_LOW])],
7719            requests: requests.clone(),
7720        }));
7721        builder.inference_router(Arc::new(StaticRouter {
7722            id: "test-router",
7723            decision: InferenceRoutingDecision::selected(
7724                "test-router",
7725                ModelSelection {
7726                    provider: "missing".to_string(),
7727                    model: "missing".to_string(),
7728                },
7729                "would route if called",
7730            ),
7731            contexts: contexts.clone(),
7732        }));
7733        let thread_root =
7734            std::env::temp_dir().join(format!("roder-routing-explicit-{}", uuid::Uuid::new_v4()));
7735        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
7736            base_path: thread_root.clone(),
7737        }));
7738        let runtime = Arc::new(
7739            Runtime::new(
7740                builder.build().unwrap(),
7741                RuntimeConfig {
7742                    default_provider: selected.provider.clone(),
7743                    default_model: selected.model.clone(),
7744                    inference_router: RuntimeInferenceRouterConfig {
7745                        enabled: true,
7746                        router_id: Some("test-router".to_string()),
7747                    },
7748                    ..RuntimeConfig::default()
7749                },
7750            )
7751            .unwrap(),
7752        );
7753        let thread_id = runtime
7754            .create_thread_with(CreateThreadRequest {
7755                title: Some("Routing explicit".to_string()),
7756                workspace: test_workspace(),
7757                workspace_id: None,
7758                root_id: None,
7759                provider: Some(selected.provider.clone()),
7760                model: Some(selected.model.clone()),
7761                tool_allowlist: Vec::new(),
7762                developer_instructions: None,
7763                external_tools: Vec::new(),
7764                selection_mode: Some(ModelSelectionMode::auto(
7765                    "test-router:default",
7766                    "test-router",
7767                    "Auto",
7768                    selected.clone(),
7769                    None,
7770                    None,
7771                )),
7772                runner: None,
7773            })
7774            .await
7775            .unwrap()
7776            .thread_id;
7777        let mut rx = runtime.subscribe_events();
7778        let turn_id = runtime
7779            .start_turn(StartTurnRequest {
7780                thread_id,
7781                message: "use explicit selection".to_string(),
7782                images: Vec::new(),
7783                provider_override: Some(selected.provider.clone()),
7784                model_override: Some(selected.model.clone()),
7785                reasoning_override: None,
7786                workspace: test_workspace(),
7787                instructions: InstructionBundle::default(),
7788                developer_context: None,
7789                task_ledger_required: false,
7790                service_tier_override: None,
7791            })
7792            .await
7793            .unwrap();
7794
7795        let mut saw_routing_event = false;
7796        tokio::time::timeout(std::time::Duration::from_secs(5), async {
7797            loop {
7798                let envelope = rx.recv().await.unwrap();
7799                if envelope.turn_id.as_deref() != Some(&turn_id) {
7800                    continue;
7801                }
7802                match envelope.event {
7803                    RoderEvent::InferenceRoutingDecision(_) => {
7804                        saw_routing_event = true;
7805                    }
7806                    RoderEvent::TurnCompleted(_) => break,
7807                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7808                    _ => {}
7809                }
7810            }
7811        })
7812        .await
7813        .unwrap();
7814
7815        assert!(!saw_routing_event);
7816        assert!(contexts.lock().unwrap().is_empty());
7817        let requests = requests.lock().unwrap();
7818        assert_eq!(requests.len(), 1);
7819        assert_eq!(requests[0].model, selected);
7820        let _ = std::fs::remove_dir_all(thread_root);
7821    }
7822
7823    #[tokio::test]
7824    async fn inference_router_is_bypassed_for_manual_selection_mode() {
7825        let requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7826        let contexts = Arc::new(StdMutex::new(Vec::<InferenceRoutingContext>::new()));
7827        let selected = ModelSelection {
7828            provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7829            model: "mock".to_string(),
7830        };
7831
7832        let mut builder = ExtensionRegistryBuilder::new();
7833        builder.inference_engine(Arc::new(RoutingCaptureEngine {
7834            id: roder_api::catalog::PROVIDER_MOCK,
7835            models: vec![routing_test_model("mock", &[REASONING_LOW])],
7836            requests: requests.clone(),
7837        }));
7838        builder.inference_router(Arc::new(StaticRouter {
7839            id: "test-router",
7840            decision: InferenceRoutingDecision::selected(
7841                "test-router",
7842                ModelSelection {
7843                    provider: "missing".to_string(),
7844                    model: "missing".to_string(),
7845                },
7846                "would route if called",
7847            ),
7848            contexts: contexts.clone(),
7849        }));
7850        let thread_root =
7851            std::env::temp_dir().join(format!("roder-routing-manual-{}", uuid::Uuid::new_v4()));
7852        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
7853            base_path: thread_root.clone(),
7854        }));
7855        let runtime = Arc::new(
7856            Runtime::new(
7857                builder.build().unwrap(),
7858                RuntimeConfig {
7859                    default_provider: selected.provider.clone(),
7860                    default_model: selected.model.clone(),
7861                    inference_router: RuntimeInferenceRouterConfig {
7862                        enabled: true,
7863                        router_id: Some("test-router".to_string()),
7864                    },
7865                    ..RuntimeConfig::default()
7866                },
7867            )
7868            .unwrap(),
7869        );
7870        let thread_id = runtime
7871            .create_thread_with(CreateThreadRequest {
7872                title: Some("Routing manual".to_string()),
7873                workspace: test_workspace(),
7874                workspace_id: None,
7875                root_id: None,
7876                provider: Some(selected.provider.clone()),
7877                model: Some(selected.model.clone()),
7878                tool_allowlist: Vec::new(),
7879                developer_instructions: None,
7880                external_tools: Vec::new(),
7881                selection_mode: Some(ModelSelectionMode::manual(
7882                    selected.provider.clone(),
7883                    selected.model.clone(),
7884                    None,
7885                )),
7886                runner: None,
7887            })
7888            .await
7889            .unwrap()
7890            .thread_id;
7891        let mut rx = runtime.subscribe_events();
7892        let turn_id = runtime
7893            .start_turn(StartTurnRequest {
7894                thread_id,
7895                message: "use selected manual model".to_string(),
7896                images: Vec::new(),
7897                provider_override: None,
7898                model_override: None,
7899                reasoning_override: None,
7900                workspace: test_workspace(),
7901                instructions: InstructionBundle::default(),
7902                developer_context: None,
7903                task_ledger_required: false,
7904                service_tier_override: None,
7905            })
7906            .await
7907            .unwrap();
7908
7909        let mut saw_routing_event = false;
7910        tokio::time::timeout(std::time::Duration::from_secs(5), async {
7911            loop {
7912                let envelope = rx.recv().await.unwrap();
7913                if envelope.turn_id.as_deref() != Some(&turn_id) {
7914                    continue;
7915                }
7916                match envelope.event {
7917                    RoderEvent::InferenceRoutingDecision(_) => {
7918                        saw_routing_event = true;
7919                    }
7920                    RoderEvent::TurnCompleted(_) => break,
7921                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7922                    _ => {}
7923                }
7924            }
7925        })
7926        .await
7927        .unwrap();
7928
7929        assert!(!saw_routing_event);
7930        assert!(contexts.lock().unwrap().is_empty());
7931        let requests = requests.lock().unwrap();
7932        assert_eq!(requests.len(), 1);
7933        assert_eq!(requests[0].model, selected);
7934        let _ = std::fs::remove_dir_all(thread_root);
7935    }
7936
7937    #[test]
7938    fn enabled_inference_router_requires_registered_router() {
7939        let mut builder = ExtensionRegistryBuilder::new();
7940        builder.inference_engine(Arc::new(FakeInferenceEngine));
7941
7942        let err = match Runtime::new(
7943            builder.build().unwrap(),
7944            RuntimeConfig {
7945                inference_router: RuntimeInferenceRouterConfig {
7946                    enabled: true,
7947                    router_id: Some("missing-router".to_string()),
7948                },
7949                ..RuntimeConfig::default()
7950            },
7951        ) {
7952            Ok(_) => panic!("runtime should reject unknown inference router"),
7953            Err(err) => err,
7954        };
7955
7956        assert!(
7957            err.to_string()
7958                .contains("inference router \"missing-router\" is not registered")
7959        );
7960    }
7961
7962    #[tokio::test]
7963    async fn model_profile_routes_request_knobs_to_next_inference() {
7964        let request = captured_profile_request(RuntimeConfig {
7965            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7966            default_model: "gpt-5.5".to_string(),
7967            model_profiles: std::collections::HashMap::from([(
7968                "gpt-5.5".to_string(),
7969                test_model_profile("gpt-5.5"),
7970            )]),
7971            ..RuntimeConfig::default()
7972        })
7973        .await;
7974
7975        let tool_names = request
7976            .tools
7977            .iter()
7978            .map(|tool| tool.name.as_str())
7979            .collect::<Vec<_>>();
7980        assert!(tool_names.contains(&"apply_patch"));
7981        assert!(tool_names.contains(&"edit"));
7982        assert!(tool_names.contains(&"multi_edit"));
7983        assert!(tool_names.contains(&"write_file"));
7984        assert_eq!(request.reasoning.level.as_deref(), Some(REASONING_LOW));
7985        assert_eq!(request.runtime.parallel_tool_calls, Some(false));
7986        assert_eq!(request.runtime.auto_compact_token_limit, Some(123_000));
7987        assert!(
7988            request
7989                .instructions
7990                .developer
7991                .as_deref()
7992                .unwrap_or_default()
7993                .contains("Use the provided context as the current working set")
7994        );
7995        assert_eq!(
7996            request
7997                .metadata
7998                .pointer("/modelProfile/schemaPolicy")
7999                .and_then(serde_json::Value::as_str),
8000            Some("standard_required_first")
8001        );
8002    }
8003
8004    #[tokio::test]
8005    async fn ultra_reasoning_reaches_transport_state_and_enables_proactive_delegation() {
8006        let request = captured_profile_request(RuntimeConfig {
8007            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8008            default_model: "gpt-5.6-sol".to_string(),
8009            reasoning: Some(REASONING_ULTRA.to_string()),
8010            ..RuntimeConfig::default()
8011        })
8012        .await;
8013
8014        assert_eq!(request.reasoning.level.as_deref(), Some(REASONING_ULTRA));
8015        let developer = request
8016            .instructions
8017            .developer
8018            .as_deref()
8019            .expect("ultra developer instructions");
8020        assert!(developer.contains("Proactive multi-agent delegation is active"));
8021    }
8022
8023    #[tokio::test]
8024    async fn lower_sol_effort_requires_explicit_multi_agent_request() {
8025        let request = captured_profile_request(RuntimeConfig {
8026            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8027            default_model: "gpt-5.6-sol".to_string(),
8028            reasoning: Some(roder_api::catalog::REASONING_MEDIUM.to_string()),
8029            ..RuntimeConfig::default()
8030        })
8031        .await;
8032
8033        let developer = request
8034            .instructions
8035            .developer
8036            .as_deref()
8037            .expect("sol developer instructions");
8038        assert!(developer.contains("Do not spawn sub-agents unless"));
8039        assert!(!developer.contains("Proactive multi-agent delegation is active"));
8040    }
8041
8042    #[tokio::test]
8043    async fn luna_does_not_receive_codex_v2_multi_agent_policy() {
8044        let request = captured_profile_request(RuntimeConfig {
8045            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8046            default_model: "gpt-5.6-luna".to_string(),
8047            reasoning: Some(roder_api::catalog::REASONING_MAX.to_string()),
8048            ..RuntimeConfig::default()
8049        })
8050        .await;
8051
8052        let developer = request
8053            .instructions
8054            .developer
8055            .as_deref()
8056            .unwrap_or_default();
8057        assert!(!developer.contains("Do not spawn sub-agents unless"));
8058        assert!(!developer.contains("Proactive multi-agent delegation is active"));
8059    }
8060
8061    #[tokio::test]
8062    async fn ultra_mode_enables_proactive_multi_agent_for_any_model() {
8063        // Luna has no Ultra effort, but ultra mode still injects proactive policy.
8064        let request = captured_profile_request(RuntimeConfig {
8065            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8066            default_model: "gpt-5.6-luna".to_string(),
8067            reasoning: Some(roder_api::catalog::REASONING_MAX.to_string()),
8068            ultra_mode: true,
8069            ..RuntimeConfig::default()
8070        })
8071        .await;
8072
8073        let developer = request
8074            .instructions
8075            .developer
8076            .as_deref()
8077            .expect("ultra mode developer instructions");
8078        assert!(developer.contains("Proactive multi-agent delegation is active"));
8079        assert!(developer.contains("spawn_agent"));
8080    }
8081
8082    #[tokio::test]
8083    async fn ultra_mode_overrides_explicit_request_only_on_sol() {
8084        // Medium Sol effort is normally explicit-request-only; ultra mode flips
8085        // it to proactive without requiring Ultra reasoning effort.
8086        let request = captured_profile_request(RuntimeConfig {
8087            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8088            default_model: "gpt-5.6-sol".to_string(),
8089            reasoning: Some(roder_api::catalog::REASONING_MEDIUM.to_string()),
8090            ultra_mode: true,
8091            ..RuntimeConfig::default()
8092        })
8093        .await;
8094
8095        let developer = request
8096            .instructions
8097            .developer
8098            .as_deref()
8099            .expect("ultra mode sol developer instructions");
8100        assert!(developer.contains("Proactive multi-agent delegation is active"));
8101        assert!(!developer.contains("Do not spawn sub-agents unless"));
8102    }
8103
8104    #[tokio::test]
8105    async fn turn_developer_context_reaches_inference_and_does_not_persist() {
8106        let captured = Arc::new(StdMutex::new(None));
8107        let mut builder = ExtensionRegistryBuilder::new();
8108        builder.inference_engine(Arc::new(CapturingEngine {
8109            request: captured.clone(),
8110        }));
8111        let runtime = Arc::new(
8112            Runtime::new(
8113                builder.build().unwrap(),
8114                RuntimeConfig {
8115                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8116                    default_model: "gpt-5.5".to_string(),
8117                    ..RuntimeConfig::default()
8118                },
8119            )
8120            .unwrap(),
8121        );
8122
8123        async fn run_turn(runtime: &Arc<Runtime>, developer_context: Option<String>) {
8124            let mut rx = runtime.subscribe_events();
8125            let turn_id = runtime
8126                .start_turn(StartTurnRequest {
8127                    thread_id: "thread-turn-context".to_string(),
8128                    message: "hello".to_string(),
8129                    images: Vec::new(),
8130                    provider_override: None,
8131                    model_override: None,
8132                    reasoning_override: None,
8133                    workspace: test_workspace(),
8134                    instructions: InstructionBundle::default(),
8135                    developer_context,
8136                    task_ledger_required: false,
8137                    service_tier_override: None,
8138                })
8139                .await
8140                .unwrap();
8141            tokio::time::timeout(std::time::Duration::from_secs(5), async {
8142                loop {
8143                    let envelope = rx.recv().await.unwrap();
8144                    if envelope.turn_id.as_deref() != Some(&turn_id) {
8145                        continue;
8146                    }
8147                    match envelope.event {
8148                        RoderEvent::TurnCompleted(_) => break,
8149                        RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8150                        _ => {}
8151                    }
8152                }
8153            })
8154            .await
8155            .unwrap();
8156        }
8157
8158        run_turn(
8159            &runtime,
8160            Some("Connected accounts: example-service.".to_string()),
8161        )
8162        .await;
8163        let request = captured.lock().unwrap().clone().unwrap();
8164        assert_eq!(
8165            request.instructions.developer_context.as_deref(),
8166            Some("Connected accounts: example-service.")
8167        );
8168
8169        // The context is per-turn only: the next turn on the same thread
8170        // without a developerContext must not see the previous one.
8171        run_turn(&runtime, None).await;
8172        let request = captured.lock().unwrap().clone().unwrap();
8173        assert_eq!(request.instructions.developer_context, None);
8174    }
8175
8176    #[tokio::test]
8177    async fn tool_search_overrides_route_to_next_inference_request() {
8178        let request = captured_profile_request(RuntimeConfig {
8179            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8180            default_model: "gpt-5.4".to_string(),
8181            tool_search: ToolSearchConfig {
8182                mode: roder_api::inference::ToolSearchMode::Auto,
8183                max_catalog_items: Some(100),
8184                ..ToolSearchConfig::default()
8185            },
8186            provider_tool_search: std::collections::HashMap::from([(
8187                roder_api::catalog::PROVIDER_MOCK.to_string(),
8188                roder_api::inference::ToolSearchConfigOverlay {
8189                    include_skills: Some(false),
8190                    provider_variant: Some(roder_api::inference::ToolSearchProviderVariant::Regex),
8191                    ..Default::default()
8192                },
8193            )]),
8194            model_tool_search: std::collections::HashMap::from([(
8195                "gpt-5.4".to_string(),
8196                roder_api::inference::ToolSearchConfigOverlay {
8197                    mode: Some(roder_api::inference::ToolSearchMode::ProviderNative),
8198                    max_catalog_items: Some(25),
8199                    provider_variant: Some(roder_api::inference::ToolSearchProviderVariant::Bm25),
8200                    ..Default::default()
8201                },
8202            )]),
8203            ..RuntimeConfig::default()
8204        })
8205        .await;
8206
8207        assert_eq!(
8208            request.runtime.tool_search.mode,
8209            roder_api::inference::ToolSearchMode::ProviderNative
8210        );
8211        assert_eq!(request.runtime.tool_search.max_catalog_items, Some(25));
8212        assert_eq!(
8213            request.runtime.tool_search.provider_variant,
8214            roder_api::inference::ToolSearchProviderVariant::Bm25
8215        );
8216    }
8217
8218    #[tokio::test]
8219    async fn context_entrypoint_hints_use_turn_workspace() {
8220        let process_workspace = runtime_test_workspace("entrypoint-process");
8221        let thread_workspace = runtime_test_workspace("entrypoint-thread");
8222        std::fs::create_dir_all(process_workspace.join("src")).unwrap();
8223        std::fs::create_dir_all(thread_workspace.join("src")).unwrap();
8224        std::fs::write(
8225            process_workspace.join("src/sidebar-thread-groups.ts"),
8226            "export const desktopLeak = true;\n",
8227        )
8228        .unwrap();
8229        std::fs::write(
8230            thread_workspace.join("src/voice-plan-feedback.ts"),
8231            "export const voicePlanFeedback = true;\n",
8232        )
8233        .unwrap();
8234
8235        let captured = Arc::new(StdMutex::new(None));
8236        let mut builder = ExtensionRegistryBuilder::new();
8237        builder.inference_engine(Arc::new(CapturingEngine {
8238            request: captured.clone(),
8239        }));
8240        builder.context_planner(Arc::new(roder_context::EntrypointContextPlanner::new(
8241            process_workspace.clone(),
8242        )));
8243        let runtime = Arc::new(
8244            Runtime::new(
8245                builder.build().unwrap(),
8246                RuntimeConfig {
8247                    workspace: Some(process_workspace.display().to_string()),
8248                    ..RuntimeConfig::default()
8249                },
8250            )
8251            .unwrap(),
8252        );
8253        let mut rx = runtime.subscribe_events();
8254
8255        let turn_id = runtime
8256            .start_turn(StartTurnRequest {
8257                thread_id: "thread-workspace-entrypoint".to_string(),
8258                message: "investigate voice plan feedback".to_string(),
8259                images: Vec::new(),
8260                provider_override: None,
8261                model_override: None,
8262                reasoning_override: None,
8263                workspace: thread_workspace.display().to_string(),
8264                instructions: crate::instructions::default_instructions(),
8265                developer_context: None,
8266                task_ledger_required: false,
8267                service_tier_override: None,
8268            })
8269            .await
8270            .unwrap();
8271
8272        tokio::time::timeout(std::time::Duration::from_secs(5), async {
8273            loop {
8274                let envelope = rx.recv().await.unwrap();
8275                if envelope.turn_id.as_deref() != Some(&turn_id) {
8276                    continue;
8277                }
8278                match envelope.event {
8279                    RoderEvent::TurnCompleted(_) => break,
8280                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8281                    _ => {}
8282                }
8283            }
8284        })
8285        .await
8286        .unwrap();
8287
8288        let request = captured.lock().unwrap().clone().expect("captured request");
8289        let transcript_text = request
8290            .transcript
8291            .iter()
8292            .map(|item| match item {
8293                TranscriptItem::UserMessage(message) => message.text.as_str(),
8294                _ => "",
8295            })
8296            .collect::<Vec<_>>()
8297            .join("\n");
8298        assert!(transcript_text.contains("src/voice-plan-feedback.ts"));
8299        assert!(!transcript_text.contains("src/sidebar-thread-groups.ts"));
8300
8301        let _ = std::fs::remove_dir_all(process_workspace);
8302        let _ = std::fs::remove_dir_all(thread_workspace);
8303    }
8304
8305    fn runtime_test_workspace(name: &str) -> std::path::PathBuf {
8306        let path =
8307            std::env::temp_dir().join(format!("roder-runtime-{name}-{}", uuid::Uuid::new_v4()));
8308        let _ = std::fs::remove_dir_all(&path);
8309        std::fs::create_dir_all(&path).unwrap();
8310        path
8311    }
8312
8313    #[tokio::test]
8314    async fn model_profile_user_model_knobs_override_profile_defaults() {
8315        let request = captured_profile_request(RuntimeConfig {
8316            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8317            default_model: "gpt-5.5".to_string(),
8318            reasoning: Some(REASONING_HIGH.to_string()),
8319            auto_compact_token_limit: Some(999),
8320            model_edit_tools: std::collections::HashMap::from([(
8321                "gpt-5.5".to_string(),
8322                EDIT_TOOL_PATCH.to_string(),
8323            )]),
8324            model_parallel_tool_calls: std::collections::HashMap::from([(
8325                "gpt-5.5".to_string(),
8326                true,
8327            )]),
8328            model_profiles: std::collections::HashMap::from([(
8329                "gpt-5.5".to_string(),
8330                test_model_profile("gpt-5.5"),
8331            )]),
8332            ..RuntimeConfig::default()
8333        })
8334        .await;
8335
8336        let tool_names = request
8337            .tools
8338            .iter()
8339            .map(|tool| tool.name.as_str())
8340            .collect::<Vec<_>>();
8341        assert!(tool_names.contains(&"apply_patch"));
8342        assert!(!tool_names.contains(&"edit"));
8343        assert_eq!(request.reasoning.level.as_deref(), Some(REASONING_HIGH));
8344        assert_eq!(request.runtime.parallel_tool_calls, Some(true));
8345        assert_eq!(request.runtime.auto_compact_token_limit, Some(999));
8346    }
8347
8348    #[tokio::test]
8349    async fn runtime_tool_allowlist_filters_advertised_tools() {
8350        let request = captured_profile_request(RuntimeConfig {
8351            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8352            default_model: "gpt-5.5".to_string(),
8353            tool_allowlist: vec!["edit".to_string()],
8354            model_profiles: std::collections::HashMap::from([(
8355                "gpt-5.5".to_string(),
8356                test_model_profile("gpt-5.5"),
8357            )]),
8358            ..RuntimeConfig::default()
8359        })
8360        .await;
8361
8362        let tool_names = request
8363            .tools
8364            .iter()
8365            .map(|tool| tool.name.as_str())
8366            .collect::<Vec<_>>();
8367        assert_eq!(tool_names, vec!["edit"]);
8368    }
8369
8370    /// Builds a store-backed runtime turn for a thread created with the given per-thread
8371    /// overrides and returns the captured inference request.
8372    async fn captured_thread_override_request(
8373        runtime: &Arc<Runtime>,
8374        requests: &Arc<StdMutex<Vec<AgentInferenceRequest>>>,
8375        tool_allowlist: Vec<String>,
8376        developer_instructions: Option<String>,
8377        external_tools: Vec<ToolSpec>,
8378    ) -> AgentInferenceRequest {
8379        let thread_id = runtime
8380            .create_thread_with(CreateThreadRequest {
8381                title: Some("Thread overrides".to_string()),
8382                workspace: test_workspace(),
8383                workspace_id: None,
8384                root_id: None,
8385                provider: None,
8386                model: None,
8387                selection_mode: None,
8388                tool_allowlist,
8389                developer_instructions,
8390                external_tools,
8391                runner: None,
8392            })
8393            .await
8394            .unwrap()
8395            .thread_id;
8396        let mut rx = runtime.subscribe_events();
8397        let turn_id = runtime
8398            .start_turn(StartTurnRequest {
8399                thread_id,
8400                message: "hello".to_string(),
8401                images: Vec::new(),
8402                provider_override: None,
8403                model_override: None,
8404                reasoning_override: None,
8405                workspace: test_workspace(),
8406                instructions: crate::default_instructions(),
8407                developer_context: None,
8408                task_ledger_required: false,
8409                service_tier_override: None,
8410            })
8411            .await
8412            .unwrap();
8413        tokio::time::timeout(std::time::Duration::from_secs(5), async {
8414            loop {
8415                let envelope = rx.recv().await.unwrap();
8416                if envelope.turn_id.as_deref() != Some(&turn_id) {
8417                    continue;
8418                }
8419                match envelope.event {
8420                    RoderEvent::TurnCompleted(_) => break,
8421                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8422                    _ => {}
8423                }
8424            }
8425        })
8426        .await
8427        .unwrap();
8428        requests.lock().unwrap().pop().expect("captured request")
8429    }
8430
8431    #[tokio::test]
8432    async fn thread_tool_allowlist_filters_only_that_thread() {
8433        let requests = Arc::new(StdMutex::new(Vec::new()));
8434        let thread_root =
8435            std::env::temp_dir().join(format!("roder-thread-allowlist-{}", uuid::Uuid::new_v4()));
8436        let mut builder = ExtensionRegistryBuilder::new();
8437        builder.inference_engine(Arc::new(SwitchCaptureEngine {
8438            requests: requests.clone(),
8439        }));
8440        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8441            base_path: thread_root.clone(),
8442        }));
8443        builder.tool_contributor(Arc::new(ProfileToolContributor));
8444        let runtime = Arc::new(
8445            Runtime::new(
8446                builder.build().unwrap(),
8447                RuntimeConfig {
8448                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8449                    default_model: "gpt-5.5".to_string(),
8450                    model_profiles: std::collections::HashMap::from([(
8451                        "gpt-5.5".to_string(),
8452                        test_model_profile("gpt-5.5"),
8453                    )]),
8454                    ..RuntimeConfig::default()
8455                },
8456            )
8457            .unwrap(),
8458        );
8459
8460        let allowlisted = captured_thread_override_request(
8461            &runtime,
8462            &requests,
8463            vec!["edit".to_string()],
8464            None,
8465            Vec::new(),
8466        )
8467        .await;
8468        let unrestricted =
8469            captured_thread_override_request(&runtime, &requests, Vec::new(), None, Vec::new())
8470                .await;
8471
8472        let allowlisted_names = allowlisted
8473            .tools
8474            .iter()
8475            .map(|tool| tool.name.as_str())
8476            .collect::<Vec<_>>();
8477        assert_eq!(allowlisted_names, vec!["edit"]);
8478        let unrestricted_names = unrestricted
8479            .tools
8480            .iter()
8481            .map(|tool| tool.name.as_str())
8482            .collect::<Vec<_>>();
8483        assert!(unrestricted_names.contains(&"edit"));
8484        assert!(unrestricted_names.len() > 1);
8485
8486        let _ = std::fs::remove_dir_all(thread_root);
8487    }
8488
8489    /// Builds a store-backed runtime whose `RuntimeConfig.tool_allowlist` is `["edit"]`.
8490    fn runtime_with_edit_allowlist(
8491        requests: &Arc<StdMutex<Vec<AgentInferenceRequest>>>,
8492        thread_root: &std::path::Path,
8493    ) -> Arc<Runtime> {
8494        let mut builder = ExtensionRegistryBuilder::new();
8495        builder.inference_engine(Arc::new(SwitchCaptureEngine {
8496            requests: requests.clone(),
8497        }));
8498        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8499            base_path: thread_root.to_path_buf(),
8500        }));
8501        builder.tool_contributor(Arc::new(ProfileToolContributor));
8502        Arc::new(
8503            Runtime::new(
8504                builder.build().unwrap(),
8505                RuntimeConfig {
8506                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8507                    default_model: "gpt-5.5".to_string(),
8508                    tool_allowlist: vec!["edit".to_string()],
8509                    model_profiles: std::collections::HashMap::from([(
8510                        "gpt-5.5".to_string(),
8511                        test_model_profile("gpt-5.5"),
8512                    )]),
8513                    ..RuntimeConfig::default()
8514                },
8515            )
8516            .unwrap(),
8517        )
8518    }
8519
8520    #[tokio::test]
8521    async fn runtime_and_thread_allowlists_intersect() {
8522        let requests = Arc::new(StdMutex::new(Vec::new()));
8523        let thread_root = std::env::temp_dir().join(format!(
8524            "roder-allowlist-intersect-{}",
8525            uuid::Uuid::new_v4()
8526        ));
8527        let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8528
8529        let request = captured_thread_override_request(
8530            &runtime,
8531            &requests,
8532            vec!["edit".to_string(), "write_file".to_string()],
8533            None,
8534            Vec::new(),
8535        )
8536        .await;
8537
8538        // The thread allowlist must not re-enable tools the runtime allowlist bans.
8539        let names = request
8540            .tools
8541            .iter()
8542            .map(|tool| tool.name.as_str())
8543            .collect::<Vec<_>>();
8544        assert_eq!(names, vec!["edit"]);
8545
8546        let _ = std::fs::remove_dir_all(thread_root);
8547    }
8548
8549    #[tokio::test]
8550    async fn route_tool_call_denies_tools_outside_allowlists() {
8551        let requests = Arc::new(StdMutex::new(Vec::new()));
8552        let thread_root =
8553            std::env::temp_dir().join(format!("roder-allowlist-dispatch-{}", uuid::Uuid::new_v4()));
8554        let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8555        let thread_id = runtime
8556            .create_thread_with(CreateThreadRequest {
8557                title: Some("Dispatch allowlist".to_string()),
8558                workspace: test_workspace(),
8559                workspace_id: None,
8560                root_id: None,
8561                provider: None,
8562                model: None,
8563                selection_mode: None,
8564                tool_allowlist: vec!["edit".to_string(), "write_file".to_string()],
8565                developer_instructions: None,
8566                external_tools: Vec::new(),
8567                runner: None,
8568            })
8569            .await
8570            .unwrap()
8571            .thread_id;
8572
8573        // write_file is registered and on the thread allowlist but banned by the runtime allowlist.
8574        let result = runtime
8575            .route_tool_call(
8576                &thread_id,
8577                &"turn-allowlist-dispatch".to_string(),
8578                roder_api::inference::ToolCallCompleted {
8579                    id: "call-1".to_string(),
8580                    name: "write_file".to_string(),
8581                    arguments: r#"{"path":"a.txt","content":"hi"}"#.to_string(),
8582                },
8583                None,
8584                None,
8585            )
8586            .await
8587            .unwrap();
8588
8589        assert!(result.is_error);
8590        assert!(
8591            result
8592                .result
8593                .contains("not permitted by the tool allowlist"),
8594            "unexpected result: {}",
8595            result.result
8596        );
8597
8598        let _ = std::fs::remove_dir_all(thread_root);
8599    }
8600
8601    #[tokio::test]
8602    async fn route_tool_calls_denies_agent_swarm_mixed_with_other_tools() {
8603        let requests = Arc::new(StdMutex::new(Vec::new()));
8604        let thread_root =
8605            std::env::temp_dir().join(format!("roder-swarm-exclusive-{}", uuid::Uuid::new_v4()));
8606        let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8607
8608        // A response that mixes agent_swarm with another tool is denied wholesale:
8609        // both calls get an error result with retry guidance, and no tool runs.
8610        let results = runtime
8611            .route_tool_calls(
8612                &"thread-swarm".to_string(),
8613                &"turn-swarm".to_string(),
8614                vec![
8615                    roder_api::inference::ToolCallCompleted {
8616                        id: "swarm-1".to_string(),
8617                        name: "agent_swarm".to_string(),
8618                        arguments: "{}".to_string(),
8619                    },
8620                    roder_api::inference::ToolCallCompleted {
8621                        id: "read-1".to_string(),
8622                        name: "read_file".to_string(),
8623                        arguments: "{}".to_string(),
8624                    },
8625                ],
8626                true,
8627                None,
8628                None,
8629            )
8630            .await
8631            .unwrap();
8632
8633        assert_eq!(results.len(), 2, "every tool_call_id must get a response");
8634        assert_eq!(results[0].id, "swarm-1");
8635        assert_eq!(results[1].id, "read-1");
8636        for result in &results {
8637            assert!(result.is_error);
8638            assert!(
8639                result.result.contains("only tool call"),
8640                "unexpected result: {}",
8641                result.result
8642            );
8643        }
8644
8645        let _ = std::fs::remove_dir_all(thread_root);
8646    }
8647
8648    #[tokio::test]
8649    async fn route_tool_calls_emits_agent_swarm_started_event() {
8650        let requests = Arc::new(StdMutex::new(Vec::new()));
8651        let thread_root =
8652            std::env::temp_dir().join(format!("roder-swarm-started-{}", uuid::Uuid::new_v4()));
8653        let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8654        let mut events = runtime.subscribe_events();
8655
8656        // A single agent_swarm call (the valid shape) emits AgentSwarmStarted on
8657        // the event bus before dispatch, with the child count parsed from args.
8658        let _ = runtime
8659            .route_tool_calls(
8660                &"thread-swarm".to_string(),
8661                &"turn-swarm".to_string(),
8662                vec![roder_api::inference::ToolCallCompleted {
8663                    id: "swarm-1".to_string(),
8664                    name: "agent_swarm".to_string(),
8665                    arguments: r#"{"description":"x","prompt_template":"Read {{item}}","items":["a.rs","b.rs"]}"#
8666                        .to_string(),
8667                }],
8668                true,
8669                None,
8670                None,
8671            )
8672            .await
8673            .unwrap();
8674
8675        let mut started_child_count = None;
8676        for _ in 0..16 {
8677            let envelope = tokio::time::timeout(std::time::Duration::from_secs(2), events.recv())
8678                .await
8679                .unwrap()
8680                .unwrap();
8681            if let RoderEvent::AgentSwarmStarted(event) = envelope.event {
8682                started_child_count = Some(event.child_count);
8683                assert_eq!(event.tool_id, "swarm-1");
8684                break;
8685            }
8686        }
8687        assert_eq!(started_child_count, Some(2));
8688
8689        let _ = std::fs::remove_dir_all(thread_root);
8690    }
8691
8692    #[test]
8693    fn agent_swarm_child_count_sums_items_and_resumes() {
8694        assert_eq!(
8695            agent_swarm_child_count(r#"{"description":"x","items":["a","b","c"]}"#),
8696            3
8697        );
8698        assert_eq!(
8699            agent_swarm_child_count(
8700                r#"{"description":"x","items":["a"],"resume_agent_ids":{"id1":"continue"}}"#
8701            ),
8702            2
8703        );
8704        // Malformed input is lenient.
8705        assert_eq!(agent_swarm_child_count("not json"), 0);
8706    }
8707
8708    #[test]
8709    fn parse_swarm_counts_reads_summary_with_omitted_buckets() {
8710        let text = "<agent_swarm_result>\n<summary>completed: 2, failed: 1</summary>\n</agent_swarm_result>";
8711        assert_eq!(parse_swarm_counts(text), Some((2, 1, 0)));
8712        let text = "<agent_swarm_result>\n<summary>completed: 0</summary>\n</agent_swarm_result>";
8713        assert_eq!(parse_swarm_counts(text), Some((0, 0, 0)));
8714        // Not a swarm result.
8715        assert_eq!(parse_swarm_counts("just text"), None);
8716    }
8717
8718    /// Signals when inference starts, then waits for `proceed` and fails the stream.
8719    struct SignalledFailureEngine {
8720        started: tokio::sync::mpsc::UnboundedSender<()>,
8721        proceed: Arc<tokio::sync::Notify>,
8722    }
8723
8724    #[async_trait::async_trait]
8725    impl InferenceEngine for SignalledFailureEngine {
8726        fn id(&self) -> String {
8727            roder_api::catalog::PROVIDER_MOCK.to_string()
8728        }
8729
8730        fn capabilities(&self) -> InferenceCapabilities {
8731            InferenceCapabilities::coding_agent_default()
8732        }
8733
8734        async fn list_models(
8735            &self,
8736            _ctx: InferenceProviderContext<'_>,
8737        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
8738            Ok(roder_api::catalog::models_for_provider(
8739                roder_api::catalog::PROVIDER_MOCK,
8740                true,
8741            ))
8742        }
8743
8744        async fn stream_turn(
8745            &self,
8746            _ctx: InferenceTurnContext<'_>,
8747            _request: AgentInferenceRequest,
8748        ) -> anyhow::Result<InferenceEventStream> {
8749            let _ = self.started.send(());
8750            self.proceed.notified().await;
8751            anyhow::bail!("engine failed mid-turn")
8752        }
8753    }
8754
8755    #[tokio::test]
8756    async fn failed_turn_sweeps_pending_external_tool_calls() {
8757        let (started_tx, mut started_rx) = tokio::sync::mpsc::unbounded_channel();
8758        let proceed = Arc::new(tokio::sync::Notify::new());
8759        let mut builder = ExtensionRegistryBuilder::new();
8760        builder.inference_engine(Arc::new(SignalledFailureEngine {
8761            started: started_tx,
8762            proceed: proceed.clone(),
8763        }));
8764        let runtime =
8765            Arc::new(Runtime::new(builder.build().unwrap(), RuntimeConfig::default()).unwrap());
8766        let mut rx = runtime.subscribe_events();
8767        let turn_id = runtime
8768            .start_turn(StartTurnRequest {
8769                thread_id: "thread-sweep".to_string(),
8770                message: "go".to_string(),
8771                images: Vec::new(),
8772                provider_override: None,
8773                model_override: None,
8774                reasoning_override: None,
8775                workspace: test_workspace(),
8776                instructions: InstructionBundle {
8777                    system: None,
8778                    developer: None,
8779                    developer_context: None,
8780                },
8781                developer_context: None,
8782                task_ledger_required: false,
8783                service_tier_override: None,
8784            })
8785            .await
8786            .unwrap();
8787        tokio::time::timeout(std::time::Duration::from_secs(5), started_rx.recv())
8788            .await
8789            .unwrap()
8790            .unwrap();
8791
8792        let (tx, _pending_rx) = oneshot::channel();
8793        runtime.pending_external_tool_calls.lock().await.insert(
8794            "exttool-sweep-test".to_string(),
8795            PendingExternalToolCall {
8796                thread_id: "thread-sweep".to_string(),
8797                turn_id: turn_id.clone(),
8798                tool_id: "call-1".to_string(),
8799                tool_name: "acme_lookup".to_string(),
8800                tx,
8801            },
8802        );
8803        proceed.notify_one();
8804
8805        let outcome = tokio::time::timeout(std::time::Duration::from_secs(5), async {
8806            loop {
8807                let envelope = rx.recv().await.unwrap();
8808                if let RoderEvent::ExternalToolCallResolved(event) = envelope.event
8809                    && event.request_id == "exttool-sweep-test"
8810                {
8811                    break event.outcome;
8812                }
8813            }
8814        })
8815        .await
8816        .expect("turn failure must resolve pending external tool calls");
8817        assert_eq!(outcome, ExternalToolCallOutcome::Cancelled);
8818        assert!(runtime.pending_external_tool_calls.lock().await.is_empty());
8819    }
8820
8821    #[tokio::test]
8822    async fn thread_developer_instructions_layer_under_harness_prompt() {
8823        let requests = Arc::new(StdMutex::new(Vec::new()));
8824        let thread_root = std::env::temp_dir().join(format!(
8825            "roder-thread-instructions-{}",
8826            uuid::Uuid::new_v4()
8827        ));
8828        let mut builder = ExtensionRegistryBuilder::new();
8829        builder.inference_engine(Arc::new(SwitchCaptureEngine {
8830            requests: requests.clone(),
8831        }));
8832        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8833            base_path: thread_root.clone(),
8834        }));
8835        builder.tool_contributor(Arc::new(ProfileToolContributor));
8836        let runtime =
8837            Arc::new(Runtime::new(builder.build().unwrap(), RuntimeConfig::default()).unwrap());
8838
8839        let request = captured_thread_override_request(
8840            &runtime,
8841            &requests,
8842            Vec::new(),
8843            Some("You are embedded in a host app.".to_string()),
8844            Vec::new(),
8845        )
8846        .await;
8847
8848        let system = request.instructions.system.expect("system instructions");
8849        assert!(system.starts_with("You are Roder"));
8850        let developer = request
8851            .instructions
8852            .developer
8853            .expect("developer instructions");
8854        assert!(developer.starts_with("You are embedded in a host app."));
8855
8856        let plain =
8857            captured_thread_override_request(&runtime, &requests, Vec::new(), None, Vec::new())
8858                .await;
8859        assert_eq!(plain.instructions.developer, None);
8860
8861        let _ = std::fs::remove_dir_all(thread_root);
8862    }
8863
8864    #[tokio::test]
8865    async fn thread_external_tools_are_advertised_and_shadow_builtins() {
8866        let requests = Arc::new(StdMutex::new(Vec::new()));
8867        let thread_root = std::env::temp_dir().join(format!(
8868            "roder-thread-external-tools-{}",
8869            uuid::Uuid::new_v4()
8870        ));
8871        let mut builder = ExtensionRegistryBuilder::new();
8872        builder.inference_engine(Arc::new(SwitchCaptureEngine {
8873            requests: requests.clone(),
8874        }));
8875        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8876            base_path: thread_root.clone(),
8877        }));
8878        builder.tool_contributor(Arc::new(ProfileToolContributor));
8879        let runtime =
8880            Arc::new(Runtime::new(builder.build().unwrap(), RuntimeConfig::default()).unwrap());
8881
8882        let external_tools = vec![
8883            ToolSpec {
8884                name: "acme_lookup".to_string(),
8885                description: "Look up Acme workspace state.".to_string(),
8886                parameters: serde_json::json!({
8887                    "type": "object",
8888                    "properties": { "query": { "type": "string" } },
8889                    "required": ["query"]
8890                }),
8891            },
8892            ToolSpec {
8893                name: "edit".to_string(),
8894                description: "Host-managed edit.".to_string(),
8895                parameters: serde_json::json!({ "type": "object" }),
8896            },
8897        ];
8898        let request =
8899            captured_thread_override_request(&runtime, &requests, Vec::new(), None, external_tools)
8900                .await;
8901
8902        let acme = request
8903            .tools
8904            .iter()
8905            .find(|tool| tool.name == "acme_lookup")
8906            .expect("external tool advertised");
8907        assert_eq!(acme.description, "Look up Acme workspace state.");
8908        assert_eq!(acme.parameters["required"][0], "query");
8909        let edits = request
8910            .tools
8911            .iter()
8912            .filter(|tool| tool.name == "edit")
8913            .collect::<Vec<_>>();
8914        assert_eq!(edits.len(), 1, "external edit shadows the builtin");
8915        assert_eq!(edits[0].description, "Host-managed edit.");
8916
8917        let plain =
8918            captured_thread_override_request(&runtime, &requests, Vec::new(), None, Vec::new())
8919                .await;
8920        assert!(plain.tools.iter().all(|tool| tool.name != "acme_lookup"));
8921        let plain_edit = plain
8922            .tools
8923            .iter()
8924            .find(|tool| tool.name == "edit")
8925            .expect("builtin edit advertised on plain thread");
8926        assert_eq!(plain_edit.description, "edit test tool");
8927
8928        let _ = std::fs::remove_dir_all(thread_root);
8929    }
8930
8931    #[tokio::test]
8932    async fn model_switch_injects_summary_and_records_profile_segments() {
8933        let requests = Arc::new(StdMutex::new(Vec::new()));
8934        let thread_root = std::env::temp_dir().join(format!(
8935            "roder-model-switch-thread-{}",
8936            uuid::Uuid::new_v4()
8937        ));
8938        let mut builder = ExtensionRegistryBuilder::new();
8939        builder.inference_engine(Arc::new(SwitchCaptureEngine {
8940            requests: requests.clone(),
8941        }));
8942        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8943            base_path: thread_root.clone(),
8944        }));
8945        builder.tool_contributor(Arc::new(ProfileToolContributor));
8946        let mut claude_profile = test_model_profile("claude-haiku-4-5-20251001");
8947        claude_profile.provider_family = ProviderFamily::Anthropic;
8948        claude_profile.edit_tool = Some(EDIT_TOOL_EDIT.to_string());
8949        let runtime = Arc::new(
8950            Runtime::new(
8951                builder.build().unwrap(),
8952                RuntimeConfig {
8953                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8954                    default_model: "gpt-5.5".to_string(),
8955                    model_profiles: std::collections::HashMap::from([
8956                        ("gpt-5.5".to_string(), test_model_profile("gpt-5.5")),
8957                        ("claude-haiku-4-5-20251001".to_string(), claude_profile),
8958                    ]),
8959                    ..RuntimeConfig::default()
8960                },
8961            )
8962            .unwrap(),
8963        );
8964        let thread_id = runtime
8965            .create_thread_with(CreateThreadRequest {
8966                title: Some("Model switch".to_string()),
8967                workspace: test_workspace(),
8968                workspace_id: None,
8969                root_id: None,
8970                provider: None,
8971                model: None,
8972                selection_mode: None,
8973                tool_allowlist: Vec::new(),
8974                developer_instructions: None,
8975                external_tools: Vec::new(),
8976                runner: None,
8977            })
8978            .await
8979            .unwrap()
8980            .thread_id;
8981        let mut rx = runtime.subscribe_events();
8982        for (message, model_override) in [
8983            ("first turn", None),
8984            ("second turn", Some("claude-haiku-4-5-20251001".to_string())),
8985        ] {
8986            let turn_id = runtime
8987                .start_turn(StartTurnRequest {
8988                    thread_id: thread_id.clone(),
8989                    message: message.to_string(),
8990                    images: Vec::new(),
8991                    provider_override: None,
8992                    model_override,
8993                    reasoning_override: None,
8994                    workspace: test_workspace(),
8995                    instructions: InstructionBundle::default(),
8996                    developer_context: None,
8997                    task_ledger_required: false,
8998                    service_tier_override: None,
8999                })
9000                .await
9001                .unwrap();
9002            tokio::time::timeout(std::time::Duration::from_secs(5), async {
9003                loop {
9004                    let envelope = rx.recv().await.unwrap();
9005                    if envelope.turn_id.as_deref() != Some(&turn_id) {
9006                        continue;
9007                    }
9008                    match envelope.event {
9009                        RoderEvent::TurnCompleted(_) => break,
9010                        RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9011                        _ => {}
9012                    }
9013                }
9014            })
9015            .await
9016            .unwrap();
9017        }
9018
9019        let captured = requests.lock().unwrap().clone();
9020        assert_eq!(captured.len(), 2);
9021        assert!(captured[1].transcript.iter().any(|item| {
9022            matches!(
9023                item,
9024                TranscriptItem::UserMessage(message)
9025                    if message.text.starts_with(MODEL_SWITCH_SUMMARY_PREFIX)
9026                        && message.text.contains("previous profile mock/gpt-5.5")
9027                        && message.text.contains("Current profile mock/claude-haiku-4-5-20251001")
9028                        && message.text.contains("Available tools now:")
9029            )
9030        }));
9031
9032        let snapshot = runtime
9033            .thread_store
9034            .as_ref()
9035            .unwrap()
9036            .load_thread(&thread_id)
9037            .await
9038            .unwrap()
9039            .unwrap();
9040        let trace_segments = snapshot
9041            .turns
9042            .iter()
9043            .flat_map(|turn| &turn.items)
9044            .filter(|item| {
9045                matches!(
9046                    item,
9047                    TranscriptItem::ProviderMetadata(value)
9048                        if value.get("kind").and_then(serde_json::Value::as_str)
9049                            == Some(MODEL_PROFILE_TRACE_KIND)
9050                            && value.get("segment").and_then(serde_json::Value::as_str)
9051                                == Some("assistant")
9052                )
9053            })
9054            .count();
9055        assert!(trace_segments >= 2);
9056        let _ = std::fs::remove_dir_all(thread_root);
9057    }
9058
9059    struct CountingTaskTool {
9060        calls: Arc<StdMutex<u32>>,
9061    }
9062
9063    #[async_trait::async_trait]
9064    impl ToolExecutor for CountingTaskTool {
9065        fn spec(&self) -> ToolSpec {
9066            ToolSpec {
9067                name: "task".to_string(),
9068                description: "Dispatch a test subagent.".to_string(),
9069                parameters: serde_json::json!({
9070                    "type": "object",
9071                    "properties": {
9072                        "description": { "type": "string" },
9073                        "prompt": { "type": "string" },
9074                        "parent_deadline_seconds": { "type": "integer" }
9075                    },
9076                    "required": ["description", "prompt"],
9077                    "additionalProperties": false
9078                }),
9079            }
9080        }
9081
9082        async fn execute(
9083            &self,
9084            _ctx: ToolExecutionContext,
9085            call: ToolCall,
9086        ) -> anyhow::Result<ToolResult> {
9087            *self.calls.lock().unwrap() += 1;
9088            Ok(ToolResult {
9089                id: call.id,
9090                name: call.name,
9091                text: "started child".to_string(),
9092                data: serde_json::json!({}),
9093                is_error: false,
9094            })
9095        }
9096    }
9097
9098    #[tokio::test]
9099    async fn runtime_profile_reaches_inference_request_and_turn_metadata() {
9100        let captured = Arc::new(StdMutex::new(None));
9101        let mut builder = ExtensionRegistryBuilder::new();
9102        builder.inference_engine(Arc::new(CapturingEngine {
9103            request: captured.clone(),
9104        }));
9105        let runtime = Arc::new(
9106            Runtime::new(
9107                builder.build().unwrap(),
9108                RuntimeConfig {
9109                    runtime_profile: RuntimeProfile::NonInteractive,
9110                    ..RuntimeConfig::default()
9111                },
9112            )
9113            .unwrap(),
9114        );
9115        let mut rx = runtime.subscribe_events();
9116        let turn_id = runtime
9117            .start_turn(StartTurnRequest {
9118                thread_id: "thread-profile".to_string(),
9119                message: "work unattended".to_string(),
9120                images: Vec::new(),
9121                provider_override: None,
9122                model_override: None,
9123                reasoning_override: None,
9124                workspace: test_workspace(),
9125                instructions: InstructionBundle {
9126                    system: None,
9127                    developer: Some("base developer".to_string()),
9128                    developer_context: None,
9129                },
9130                developer_context: None,
9131                task_ledger_required: false,
9132                service_tier_override: None,
9133            })
9134            .await
9135            .unwrap();
9136
9137        let mut observed_profile = None;
9138        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9139            loop {
9140                let envelope = rx.recv().await.unwrap();
9141                if envelope.turn_id.as_deref() != Some(&turn_id) {
9142                    continue;
9143                }
9144                match envelope.event {
9145                    RoderEvent::TurnStarted(event) => {
9146                        observed_profile = Some(event.runtime_profile);
9147                    }
9148                    RoderEvent::TurnCompleted(_) => break,
9149                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9150                    _ => {}
9151                }
9152            }
9153        })
9154        .await
9155        .unwrap();
9156
9157        assert_eq!(observed_profile, Some(RuntimeProfile::NonInteractive));
9158        let request = captured.lock().unwrap().clone().unwrap();
9159        assert_eq!(request.runtime.profile, RuntimeProfile::NonInteractive);
9160        let developer = request.instructions.developer.unwrap();
9161        assert!(developer.contains("base developer"));
9162        assert!(developer.contains("non-interactive profile"));
9163    }
9164
9165    #[tokio::test]
9166    async fn global_policy_mode_changes_do_not_create_runtime_thread_directory() {
9167        let workspace = runtime_test_workspace("global-policy-mode");
9168        let thread_root = workspace.join("threads");
9169        let mut builder = ExtensionRegistryBuilder::new();
9170        builder.inference_engine(Arc::new(FakeInferenceEngine));
9171        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
9172            base_path: thread_root.clone(),
9173        }));
9174        let runtime = Runtime::new(
9175            builder.build().unwrap(),
9176            RuntimeConfig {
9177                workspace: Some(workspace.display().to_string()),
9178                ..Default::default()
9179            },
9180        )
9181        .unwrap();
9182
9183        runtime
9184            .set_policy_mode(PolicyMode::AcceptAll, Some("test".to_string()))
9185            .await
9186            .unwrap();
9187
9188        assert!(!thread_root.join("runtime").exists());
9189        let _ = std::fs::remove_dir_all(workspace);
9190    }
9191
9192    #[tokio::test]
9193    async fn task_ledger_enforcement_injects_eval_reminder_before_work() {
9194        let captured = Arc::new(StdMutex::new(None));
9195        let mut builder = ExtensionRegistryBuilder::new();
9196        builder.inference_engine(Arc::new(CapturingEngine {
9197            request: captured.clone(),
9198        }));
9199        builder.tool_contributor(Arc::new(
9200            roder_ext_task_ledger::TaskLedgerToolContributor::default(),
9201        ));
9202        let runtime = Arc::new(
9203            Runtime::new(
9204                builder.build().unwrap(),
9205                RuntimeConfig {
9206                    runtime_profile: RuntimeProfile::Eval,
9207                    policy_mode: PolicyMode::Bypass,
9208                    agent_swarm_mode: false,
9209                    ultra_mode: false,
9210                    ..RuntimeConfig::default()
9211                },
9212            )
9213            .unwrap(),
9214        );
9215        let mut rx = runtime.subscribe_events();
9216        let turn_id = runtime
9217            .start_turn(StartTurnRequest {
9218                thread_id: "thread-ledger".to_string(),
9219                message: "decomposed work".to_string(),
9220                images: Vec::new(),
9221                provider_override: None,
9222                model_override: None,
9223                reasoning_override: None,
9224                workspace: test_workspace(),
9225                instructions: InstructionBundle::default(),
9226                developer_context: None,
9227                task_ledger_required: true,
9228                service_tier_override: None,
9229            })
9230            .await
9231            .unwrap();
9232
9233        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9234            loop {
9235                let envelope = rx.recv().await.unwrap();
9236                if envelope.turn_id.as_deref() == Some(&turn_id)
9237                    && matches!(envelope.event, RoderEvent::TurnCompleted(_))
9238                {
9239                    break;
9240                }
9241            }
9242        })
9243        .await
9244        .unwrap();
9245
9246        let request = captured.lock().unwrap().clone().unwrap();
9247        let developer = request.instructions.developer.unwrap();
9248        assert!(developer.contains("Task Ledger Required"));
9249        assert!(developer.contains("task_ledger.update"));
9250        let tool_names: Vec<_> = request
9251            .tools
9252            .iter()
9253            .map(|tool| tool.name.as_str())
9254            .collect();
9255        assert!(
9256            tool_names.contains(&TASK_LEDGER_TOOL_NAME),
9257            "tool names: {tool_names:?}"
9258        );
9259        assert_eq!(
9260            request.tool_choice,
9261            ToolChoice::Specific(TASK_LEDGER_TOOL_NAME.to_string())
9262        );
9263        assert_eq!(request.tools.len(), 1);
9264        assert_eq!(request.tools[0].name, TASK_LEDGER_TOOL_NAME);
9265    }
9266
9267    #[tokio::test]
9268    async fn eval_task_ledger_blocks_final_answer_until_open_items_are_completed() {
9269        let requests = Arc::new(StdMutex::new(Vec::new()));
9270        let mut builder = ExtensionRegistryBuilder::new();
9271        builder.inference_engine(Arc::new(TaskLedgerCompletionGateEngine {
9272            calls: StdMutex::new(0),
9273            requests: requests.clone(),
9274        }));
9275        builder.tool_contributor(Arc::new(
9276            roder_ext_task_ledger::TaskLedgerToolContributor::default(),
9277        ));
9278        let runtime = Arc::new(
9279            Runtime::new(
9280                builder.build().unwrap(),
9281                RuntimeConfig {
9282                    runtime_profile: RuntimeProfile::Eval,
9283                    policy_mode: PolicyMode::Bypass,
9284                    agent_swarm_mode: false,
9285                    ultra_mode: false,
9286                    ..RuntimeConfig::default()
9287                },
9288            )
9289            .unwrap(),
9290        );
9291        let mut rx = runtime.subscribe_events();
9292        let turn_id = runtime
9293            .start_turn(StartTurnRequest {
9294                thread_id: "thread-ledger-completion".to_string(),
9295                message: "write the answer file".to_string(),
9296                images: Vec::new(),
9297                provider_override: None,
9298                model_override: None,
9299                reasoning_override: None,
9300                workspace: test_workspace(),
9301                instructions: InstructionBundle::default(),
9302                developer_context: None,
9303                task_ledger_required: true,
9304                service_tier_override: None,
9305            })
9306            .await
9307            .unwrap();
9308
9309        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9310            loop {
9311                let envelope = rx.recv().await.unwrap();
9312                if envelope.turn_id.as_deref() != Some(&turn_id) {
9313                    continue;
9314                }
9315                match envelope.event {
9316                    RoderEvent::TurnCompleted(_) => break,
9317                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9318                    _ => {}
9319                }
9320            }
9321        })
9322        .await
9323        .unwrap();
9324
9325        let requests = requests.lock().unwrap().clone();
9326        assert_eq!(requests.len(), 4);
9327        assert!(requests[2].transcript.iter().any(|item| {
9328            matches!(
9329                item,
9330                TranscriptItem::UserMessage(message)
9331                    if message.text.contains("Task Ledger Completion Required")
9332                        && message.text.contains("Write /app/result.txt")
9333            )
9334        }));
9335        assert!(requests[3].transcript.iter().any(|item| {
9336            matches!(
9337                item,
9338                TranscriptItem::ToolResult(result)
9339                    if result.name.as_deref() == Some(TASK_LEDGER_TOOL_NAME)
9340                        && result.result.contains("Task ledger: 2/2 completed")
9341            )
9342        }));
9343    }
9344
9345    #[tokio::test]
9346    async fn eval_task_ledger_checkpoint_requests_scoreable_file_before_final_reserve() {
9347        let requests = Arc::new(StdMutex::new(Vec::new()));
9348        let mut builder = ExtensionRegistryBuilder::new();
9349        builder.inference_engine(Arc::new(TaskLedgerCompletionGateEngine {
9350            calls: StdMutex::new(0),
9351            requests: requests.clone(),
9352        }));
9353        builder.tool_contributor(Arc::new(
9354            roder_ext_task_ledger::TaskLedgerToolContributor::default(),
9355        ));
9356        let runtime = Arc::new(
9357            Runtime::new(
9358                builder.build().unwrap(),
9359                RuntimeConfig {
9360                    runtime_profile: RuntimeProfile::Eval,
9361                    policy_mode: PolicyMode::Bypass,
9362                    agent_swarm_mode: false,
9363                    ultra_mode: false,
9364                    turn_deadline_seconds: Some(120),
9365                    ..RuntimeConfig::default()
9366                },
9367            )
9368            .unwrap(),
9369        );
9370        let mut rx = runtime.subscribe_events();
9371        let turn_id = runtime
9372            .start_turn(StartTurnRequest {
9373                thread_id: "thread-ledger-checkpoint".to_string(),
9374                message: "write the answer file".to_string(),
9375                images: Vec::new(),
9376                provider_override: None,
9377                model_override: None,
9378                reasoning_override: None,
9379                workspace: test_workspace(),
9380                instructions: InstructionBundle::default(),
9381                developer_context: None,
9382                task_ledger_required: true,
9383                service_tier_override: None,
9384            })
9385            .await
9386            .unwrap();
9387
9388        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9389            loop {
9390                let envelope = rx.recv().await.unwrap();
9391                if envelope.turn_id.as_deref() != Some(&turn_id) {
9392                    continue;
9393                }
9394                match envelope.event {
9395                    RoderEvent::TurnCompleted(_) => break,
9396                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9397                    _ => {}
9398                }
9399            }
9400        })
9401        .await
9402        .unwrap();
9403
9404        let requests = requests.lock().unwrap().clone();
9405        assert!(requests.len() >= 2);
9406        assert!(requests[1].transcript.iter().any(|item| {
9407            matches!(
9408                item,
9409                TranscriptItem::UserMessage(message)
9410                    if message.text.contains("Scoreable Output Checkpoint")
9411                        && message.text.contains("ensure the required output file(s) exist")
9412                        && message.text.contains("Write /app/result.txt")
9413            )
9414        }));
9415    }
9416
9417    #[test]
9418    fn deadline_task_ledger_prompt_preserves_scoreable_work_instruction() {
9419        let prompt = task_ledger_deadline_completion_prompt(
9420            12,
9421            30,
9422            "Task Ledger Completion Required: write /app/result.txt, then call task_ledger.update",
9423        );
9424
9425        assert!(prompt.contains("12 seconds remain"));
9426        assert!(prompt.contains("create or update the required scoreable output files"));
9427        assert!(prompt.contains("write /app/result.txt"));
9428        assert!(prompt.contains(TASK_LEDGER_TOOL_NAME));
9429    }
9430
9431    #[test]
9432    fn scoreable_checkpoint_prompt_preserves_provisional_file_instruction() {
9433        let prompt = task_ledger_scoreable_checkpoint_prompt(
9434            120,
9435            "Task Ledger Completion Required: write /app/result.txt, then call task_ledger.update",
9436        );
9437
9438        assert!(prompt.contains("120 seconds remain"));
9439        assert!(prompt.contains("best evidence-backed answer"));
9440        assert!(prompt.contains("even if provisional"));
9441        assert!(prompt.contains("preserve that candidate"));
9442        assert!(prompt.contains("partial-coverage"));
9443        assert!(prompt.contains("write /app/result.txt"));
9444        assert!(prompt.contains(TASK_LEDGER_TOOL_NAME));
9445    }
9446
9447    #[test]
9448    fn open_task_ledger_moves_inference_timeout_to_scoreable_checkpoint() {
9449        let deadline = Some(OffsetDateTime::now_utc() + Duration::seconds(870));
9450        let transcript = vec![TranscriptItem::ToolResult(ToolResultRecord {
9451            id: "ledger-open".to_string(),
9452            name: Some(TASK_LEDGER_TOOL_NAME.to_string()),
9453            result: "Task ledger: 0/1 completed\n- pending: Write /app/result.txt [write]"
9454                .to_string(),
9455            display_payload: None,
9456            is_error: false,
9457        })];
9458
9459        let (_, action) = inference_timeout_deadline(
9460            deadline,
9461            RuntimeProfile::Eval,
9462            true,
9463            30,
9464            false,
9465            0,
9466            &transcript,
9467        )
9468        .unwrap();
9469
9470        assert_eq!(action, InferenceTimeoutAction::ScoreableCheckpoint);
9471    }
9472
9473    #[tokio::test]
9474    async fn verification_gate_forces_eval_code_changes_through_review() {
9475        let mut builder = ExtensionRegistryBuilder::new();
9476        builder.inference_engine(Arc::new(VerificationGateEngine {
9477            calls: StdMutex::new(0),
9478        }));
9479        builder.tool_contributor(Arc::new(WriteFileContributor));
9480        builder.tool_contributor(Arc::new(
9481            roder_ext_verification::VerificationToolContributor,
9482        ));
9483        let runtime = Arc::new(
9484            Runtime::new(
9485                builder.build().unwrap(),
9486                RuntimeConfig {
9487                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9488                    default_model: "mock".to_string(),
9489                    runtime_profile: RuntimeProfile::Eval,
9490                    policy_mode: PolicyMode::Bypass,
9491                    agent_swarm_mode: false,
9492                    ultra_mode: false,
9493                    ..RuntimeConfig::default()
9494                },
9495            )
9496            .unwrap(),
9497        );
9498        let mut rx = runtime.subscribe_events();
9499        let turn_id = runtime
9500            .start_turn(StartTurnRequest {
9501                thread_id: "thread-verification".to_string(),
9502                message: "write code".to_string(),
9503                images: Vec::new(),
9504                provider_override: None,
9505                model_override: None,
9506                reasoning_override: None,
9507                workspace: test_workspace(),
9508                instructions: InstructionBundle::default(),
9509                developer_context: None,
9510                task_ledger_required: false,
9511                service_tier_override: None,
9512            })
9513            .await
9514            .unwrap();
9515
9516        let mut saw_required = false;
9517        let mut saw_completed = false;
9518        let mut final_text = String::new();
9519        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9520            loop {
9521                let envelope = rx.recv().await.unwrap();
9522                if envelope.turn_id.as_deref() != Some(&turn_id) {
9523                    continue;
9524                }
9525                match envelope.event {
9526                    RoderEvent::VerificationRequired(event) => {
9527                        saw_required = true;
9528                        assert_eq!(event.changed_files, vec!["src/lib.rs"]);
9529                    }
9530                    RoderEvent::VerificationCompleted(event) => {
9531                        saw_completed = true;
9532                        assert!(event.passed);
9533                    }
9534                    RoderEvent::InferenceEventReceived(event) => {
9535                        if let InferenceEvent::MessageDelta(delta) = event.event {
9536                            final_text.push_str(&delta.text);
9537                        }
9538                    }
9539                    RoderEvent::TurnCompleted(_) => break,
9540                    _ => {}
9541                }
9542            }
9543        })
9544        .await
9545        .unwrap();
9546
9547        assert!(saw_required);
9548        assert!(saw_completed);
9549        assert!(final_text.contains("verified final"));
9550    }
9551
9552    #[tokio::test]
9553    async fn speed_policy_changes_reasoning_across_eval_model_calls_without_model_switch() {
9554        let requests = Arc::new(StdMutex::new(Vec::new()));
9555        let mut builder = ExtensionRegistryBuilder::new();
9556        builder.inference_engine(Arc::new(SpeedPolicyEngine {
9557            calls: StdMutex::new(0),
9558            requests: requests.clone(),
9559        }));
9560        builder.tool_contributor(Arc::new(WriteFileContributor));
9561        builder.tool_contributor(Arc::new(
9562            roder_ext_verification::VerificationToolContributor,
9563        ));
9564        let runtime = Arc::new(
9565            Runtime::new(
9566                builder.build().unwrap(),
9567                RuntimeConfig {
9568                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9569                    default_model: "gpt-5.5".to_string(),
9570                    runtime_profile: RuntimeProfile::Eval,
9571                    policy_mode: PolicyMode::Bypass,
9572                    agent_swarm_mode: false,
9573                    ultra_mode: false,
9574                    ..RuntimeConfig::default()
9575                },
9576            )
9577            .unwrap(),
9578        );
9579        let mut rx = runtime.subscribe_events();
9580        let turn_id = runtime
9581            .start_turn(StartTurnRequest {
9582                thread_id: "thread-speed-policy".to_string(),
9583                message: "write code".to_string(),
9584                images: Vec::new(),
9585                provider_override: None,
9586                model_override: None,
9587                reasoning_override: None,
9588                workspace: test_workspace(),
9589                instructions: InstructionBundle::default(),
9590                developer_context: None,
9591                task_ledger_required: false,
9592                service_tier_override: None,
9593            })
9594            .await
9595            .unwrap();
9596
9597        let mut saw_speed_policy_event = false;
9598        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9599            loop {
9600                let envelope = rx.recv().await.unwrap();
9601                if envelope.turn_id.as_deref() != Some(&turn_id) {
9602                    continue;
9603                }
9604                match envelope.event {
9605                    RoderEvent::InferenceStarted(event) => {
9606                        if event.speed_policy.is_some() {
9607                            saw_speed_policy_event = true;
9608                        }
9609                    }
9610                    RoderEvent::TurnCompleted(_) => break,
9611                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9612                    _ => {}
9613                }
9614            }
9615        })
9616        .await
9617        .unwrap();
9618
9619        let requests = requests.lock().unwrap().clone();
9620        assert!(saw_speed_policy_event);
9621        assert!(requests.len() >= 4);
9622        assert!(requests.iter().all(|request| {
9623            request.model.provider == roder_api::catalog::PROVIDER_MOCK
9624                && request.model.model == "gpt-5.5"
9625        }));
9626        assert_eq!(
9627            requests[0].runtime.speed_policy.as_ref().map(|d| d.phase),
9628            Some(roder_api::inference::SpeedPolicyPhase::Orientation)
9629        );
9630        assert_eq!(requests[0].reasoning.level.as_deref(), Some(REASONING_HIGH));
9631        assert_eq!(
9632            requests[1].runtime.speed_policy.as_ref().map(|d| d.phase),
9633            Some(roder_api::inference::SpeedPolicyPhase::Execution)
9634        );
9635        assert_eq!(requests[1].reasoning.level.as_deref(), Some(REASONING_LOW));
9636        assert_eq!(
9637            requests[2].runtime.speed_policy.as_ref().map(|d| d.phase),
9638            Some(roder_api::inference::SpeedPolicyPhase::Verification)
9639        );
9640        assert_eq!(requests[2].reasoning.level.as_deref(), Some(REASONING_HIGH));
9641        assert_eq!(
9642            requests[2]
9643                .metadata
9644                .pointer("/speedPolicy/phase")
9645                .and_then(serde_json::Value::as_str),
9646            Some("verification")
9647        );
9648    }
9649
9650    #[tokio::test]
9651    async fn deadline_turn_timeout_emits_partial_result_and_clears_active_turn() {
9652        let mut builder = ExtensionRegistryBuilder::new();
9653        builder.inference_engine(Arc::new(DeadlineEngine));
9654        let runtime = Arc::new(
9655            Runtime::new(
9656                builder.build().unwrap(),
9657                RuntimeConfig {
9658                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9659                    default_model: "mock".to_string(),
9660                    runtime_profile: RuntimeProfile::Eval,
9661                    turn_deadline_seconds: Some(1),
9662                    ..RuntimeConfig::default()
9663                },
9664            )
9665            .unwrap(),
9666        );
9667        let mut rx = runtime.subscribe_events();
9668        let turn_id = runtime
9669            .start_turn(StartTurnRequest {
9670                thread_id: "thread-deadline".to_string(),
9671                message: "slow work".to_string(),
9672                images: Vec::new(),
9673                provider_override: None,
9674                model_override: None,
9675                reasoning_override: None,
9676                workspace: test_workspace(),
9677                instructions: InstructionBundle::default(),
9678                developer_context: None,
9679                task_ledger_required: false,
9680                service_tier_override: None,
9681            })
9682            .await
9683            .unwrap();
9684
9685        let mut saw_partial = false;
9686        let mut saw_deadline = false;
9687        let mut failed_kind = None;
9688        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9689            loop {
9690                let envelope = rx.recv().await.unwrap();
9691                if envelope.turn_id.as_deref() != Some(&turn_id) {
9692                    continue;
9693                }
9694                match envelope.event {
9695                    RoderEvent::TurnPartialResult(event) => {
9696                        saw_partial = event.summary.contains("partial turn state");
9697                    }
9698                    RoderEvent::TurnDeadlineExceeded(event) => {
9699                        saw_deadline = event.partial_result.contains("transcript items");
9700                    }
9701                    RoderEvent::TurnFailed(event) => {
9702                        failed_kind = event.error_kind;
9703                        break;
9704                    }
9705                    _ => {}
9706                }
9707            }
9708        })
9709        .await
9710        .unwrap();
9711
9712        for _ in 0..20 {
9713            if !runtime.active_turns.read().await.contains_key(&turn_id) {
9714                break;
9715            }
9716            tokio::time::sleep(std::time::Duration::from_millis(10)).await;
9717        }
9718        assert!(saw_partial);
9719        assert!(saw_deadline);
9720        assert_eq!(failed_kind.as_deref(), Some("deadline_timeout"));
9721        assert!(!runtime.active_turns.read().await.contains_key(&turn_id));
9722    }
9723
9724    #[tokio::test]
9725    async fn deadline_skips_subagent_task_when_remaining_budget_is_too_low() {
9726        let calls = Arc::new(StdMutex::new(0));
9727        let mut builder = ExtensionRegistryBuilder::new();
9728        builder.inference_engine(Arc::new(CapturingEngine {
9729            request: Arc::new(StdMutex::new(None)),
9730        }));
9731        let task_tool = Arc::new(CountingTaskTool {
9732            calls: calls.clone(),
9733        });
9734        builder.tool_contributor(Arc::new(TestToolContributor { tool: task_tool }));
9735        let runtime = Arc::new(
9736            Runtime::new(
9737                builder.build().unwrap(),
9738                RuntimeConfig {
9739                    policy_mode: PolicyMode::Bypass,
9740                    agent_swarm_mode: false,
9741                    ultra_mode: false,
9742                    ..RuntimeConfig::default()
9743                },
9744            )
9745            .unwrap(),
9746        );
9747
9748        let result = runtime
9749            .route_tool_call(
9750                &"thread-deadline-task".to_string(),
9751                &"turn-deadline-task".to_string(),
9752                ToolCallCompleted {
9753                    id: "task-1".to_string(),
9754                    name: "task".to_string(),
9755                    arguments: serde_json::json!({
9756                        "description": "inspect",
9757                        "prompt": "read"
9758                    })
9759                    .to_string(),
9760                },
9761                None,
9762                Some(OffsetDateTime::now_utc() + Duration::seconds(1)),
9763            )
9764            .await
9765            .unwrap();
9766
9767        assert!(result.is_error);
9768        assert!(result.result.contains("deadline policy skipped"));
9769        assert_eq!(*calls.lock().unwrap(), 0);
9770    }
9771
9772    struct TestToolContributor {
9773        tool: Arc<dyn ToolExecutor>,
9774    }
9775
9776    impl ToolContributor for TestToolContributor {
9777        fn id(&self) -> String {
9778            "test-tool".to_string()
9779        }
9780
9781        fn contribute(&self, registry: &mut ToolRegistry) -> anyhow::Result<()> {
9782            registry.register(self.tool.clone())
9783        }
9784    }
9785
9786    #[tokio::test]
9787    async fn agent_swarm_mode_override_is_per_thread() {
9788        let runtime = Runtime::fake().unwrap();
9789        let trigger = roder_api::subagents::AgentSwarmModeTrigger::Manual;
9790
9791        // Off everywhere by default.
9792        assert!(
9793            !runtime
9794                .effective_agent_swarm_mode_for_thread("thread-a")
9795                .await
9796        );
9797        assert!(
9798            !runtime
9799                .effective_agent_swarm_mode_for_thread("thread-b")
9800                .await
9801        );
9802
9803        // Enabling on thread-a does not leak into thread-b.
9804        assert!(
9805            runtime
9806                .set_agent_swarm_mode_for_thread("thread-a", true, trigger)
9807                .await
9808        );
9809        assert!(
9810            runtime
9811                .effective_agent_swarm_mode_for_thread("thread-a")
9812                .await
9813        );
9814        assert!(
9815            !runtime
9816                .effective_agent_swarm_mode_for_thread("thread-b")
9817                .await
9818        );
9819
9820        // A per-thread `off` override wins over a runtime-global `on` default.
9821        runtime.set_agent_swarm_mode(true, trigger).await.unwrap();
9822        runtime
9823            .set_agent_swarm_mode_for_thread("thread-b", false, trigger)
9824            .await;
9825        assert!(
9826            !runtime
9827                .effective_agent_swarm_mode_for_thread("thread-b")
9828                .await,
9829            "explicit per-thread off overrides the global on default"
9830        );
9831        // A thread with no override still follows the global default.
9832        assert!(
9833            runtime
9834                .effective_agent_swarm_mode_for_thread("thread-c")
9835                .await,
9836            "threads without an override follow the runtime-global default"
9837        );
9838    }
9839
9840    #[tokio::test]
9841    async fn set_agent_swarm_mode_for_thread_emits_event_with_real_thread_id() {
9842        let runtime = Runtime::fake().unwrap();
9843        let mut events = runtime.subscribe_events();
9844        runtime
9845            .set_agent_swarm_mode_for_thread(
9846                "thread-xyz",
9847                true,
9848                roder_api::subagents::AgentSwarmModeTrigger::Task,
9849            )
9850            .await;
9851        let mut saw = false;
9852        for _ in 0..8 {
9853            let envelope = tokio::time::timeout(std::time::Duration::from_secs(2), events.recv())
9854                .await
9855                .unwrap()
9856                .unwrap();
9857            if let RoderEvent::AgentSwarmModeChanged(event) = envelope.event {
9858                assert_eq!(event.thread_id, "thread-xyz");
9859                assert!(event.enabled);
9860                assert_eq!(
9861                    event.trigger,
9862                    roder_api::subagents::AgentSwarmModeTrigger::Task
9863                );
9864                saw = true;
9865                break;
9866            }
9867        }
9868        assert!(
9869            saw,
9870            "expected an AgentSwarmModeChanged event for the thread"
9871        );
9872    }
9873
9874    #[tokio::test]
9875    async fn ultra_mode_override_is_per_thread() {
9876        let runtime = Runtime::fake().unwrap();
9877        let trigger = roder_api::subagents::UltraModeTrigger::Manual;
9878
9879        assert!(!runtime.effective_ultra_mode_for_thread("thread-a").await);
9880        assert!(!runtime.effective_ultra_mode_for_thread("thread-b").await);
9881
9882        assert!(
9883            runtime
9884                .set_ultra_mode_for_thread("thread-a", true, trigger)
9885                .await
9886        );
9887        assert!(runtime.effective_ultra_mode_for_thread("thread-a").await);
9888        assert!(!runtime.effective_ultra_mode_for_thread("thread-b").await);
9889
9890        runtime.set_ultra_mode(true, trigger).await.unwrap();
9891        runtime
9892            .set_ultra_mode_for_thread("thread-b", false, trigger)
9893            .await;
9894        assert!(
9895            !runtime.effective_ultra_mode_for_thread("thread-b").await,
9896            "explicit per-thread off overrides the global on default"
9897        );
9898        assert!(
9899            runtime.effective_ultra_mode_for_thread("thread-c").await,
9900            "threads without an override follow the runtime-global default"
9901        );
9902    }
9903
9904    #[tokio::test]
9905    async fn set_ultra_mode_for_thread_emits_event_with_real_thread_id() {
9906        let runtime = Runtime::fake().unwrap();
9907        let mut events = runtime.subscribe_events();
9908        runtime
9909            .set_ultra_mode_for_thread(
9910                "thread-ultra",
9911                true,
9912                roder_api::subagents::UltraModeTrigger::Task,
9913            )
9914            .await;
9915        let mut saw = false;
9916        for _ in 0..8 {
9917            let envelope = tokio::time::timeout(std::time::Duration::from_secs(2), events.recv())
9918                .await
9919                .unwrap()
9920                .unwrap();
9921            if let RoderEvent::UltraModeChanged(event) = envelope.event {
9922                assert_eq!(event.thread_id, "thread-ultra");
9923                assert!(event.enabled);
9924                assert_eq!(event.trigger, roder_api::subagents::UltraModeTrigger::Task);
9925                saw = true;
9926                break;
9927            }
9928        }
9929        assert!(saw, "expected an UltraModeChanged event for the thread");
9930    }
9931}
9932
9933#[cfg(test)]
9934#[path = "runtime/mailbox_test_helpers.rs"]
9935mod mailbox_test_helpers;