Skip to main content

roder_core/
runtime.rs

1#[path = "runtime/sampling.rs"]
2pub(crate) mod sampling;
3#[path = "tool_advertisement.rs"]
4mod tool_advertisement;
5use sampling::{
6    SamplingPreempted, SamplingRetry, completed_call_replayed, preemptible, wait_for_steer,
7};
8#[path = "runtime/sampling_output.rs"]
9mod sampling_output;
10use sampling_output::{OutputContext, SamplingOutput};
11mod compaction_template;
12#[path = "runtime/eager_tools.rs"]
13mod eager_tools;
14mod execution_receipts;
15mod owner_handoff;
16mod service_tier;
17mod thread_admission;
18#[path = "runtime/tool_approvals.rs"]
19mod tool_approvals;
20#[path = "runtime/turn_start.rs"]
21mod turn_start;
22#[path = "runtime/usage_accounting.rs"]
23mod usage_accounting;
24use eager_tools::EagerTools;
25#[path = "runtime/patch_progress.rs"]
26mod patch_progress;
27use patch_progress::PatchProgressTracker;
28
29use std::collections::{HashMap, HashSet};
30use std::path::PathBuf;
31use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
32use std::sync::{Arc, Weak};
33
34use anyhow::Context;
35use futures::StreamExt;
36use futures::future::{AbortHandle, Abortable, BoxFuture, try_join_all};
37use roder_api::catalog::{
38    EDIT_TOOL_EDIT, EDIT_TOOL_PATCH, PROVIDER_GEMINI, REASONING_NONE, REASONING_ULTRA,
39    built_in_model_profile, built_in_model_profile_for_provider, lookup_model,
40    model_supports_reasoning_effort,
41};
42use roder_api::context::PolicyGate;
43use roder_api::events::*;
44use roder_api::extension::ExtensionRegistry;
45use roder_api::inference::{
46    AgentInferenceRequest, HostedWebSearchConfig, HostedWebSearchMode, InferenceEngine,
47    InferenceEvent, InferenceTurnContext, InstructionBundle, ModelHarnessProfile,
48    ModelSchemaPolicy, ModelSelection, OutputConfig, ProviderTurnCleanup, ReasoningConfig,
49    RuntimeHints, RuntimeProfile, TokenUsage, ToolCallCompleted, ToolSearchConfig,
50    ToolSearchConfigOverlay, finish_reason_from_stop_reason,
51};
52use roder_api::inference_routing::{InferenceRoutingOutcome, ModelSelectionMode};
53use roder_api::lifecycle::{
54    LifecycleMetricsSnapshot, TurnCleanupOwnership, TurnCleanupState, TurnLifecycleReason,
55    TurnLifecycleRecord, TurnLifecycleSnapshot, TurnLifecycleState, turn_lifecycle_snapshot,
56};
57use roder_api::policy_mode::{PolicyDecision, PolicyMode};
58use roder_api::reliability::{
59    ReliabilityContext, ReliabilityDetails, ReliabilityErrorClass, ReliabilityLimitDecision,
60    ReliabilityLimitRecorded, ReliabilityRequestPolicy, ReliabilityRetryDecision,
61    ReliabilityRetryRecorded, provider_retry_delay_ms,
62};
63use roder_api::remote_runner::{
64    RemoteRunnerProvider, RemoteRunnerSession, RemoteWorkspace, RunnerDestination,
65    RunnerSessionState, ThreadRunnerBinding,
66};
67use roder_api::subagents::SubagentDefinition;
68use roder_api::teams::{
69    TeamId, TeamMailboxMessage, TeamMailboxMessageKind, TeamMemberDescriptor, TeamMemberRole,
70    TeamMemberStatus,
71};
72use roder_api::thread::{
73    ThreadItemEvent, ThreadItemEventKind, ThreadMetadata, ThreadSnapshot, ThreadStore,
74    ThreadUsageMetadata, is_synthetic_event_thread_id, validate_thread_workspace,
75};
76use roder_api::tools::{ToolCall, ToolChoice, ToolExecutionContext, ToolRegistry, ToolResult};
77use roder_api::transcript::{
78    AssistantMessage, ErrorRecord, InputImage, ReasoningSummary, ToolCallRecord, ToolResultRecord,
79    TranscriptItem, UserMessage,
80};
81use roder_sandbox::ScopedFilesystem;
82use roder_sandbox::process::LocalProcessRunner;
83use roder_skills::{SkillRegistry, SkillRegistryOptions};
84use time::{Duration, OffsetDateTime};
85use tokio::sync::{Mutex, Notify, RwLock, oneshot};
86
87mod codex_v2;
88
89use crate::artifacts::{
90    ContextArtifactStore as FilesystemContextArtifactStore, default_context_artifact_dir,
91};
92use crate::browser_routing::apply_browser_routing;
93use crate::bus::EventBus;
94use crate::dynamic_workflows::{
95    DynamicWorkflowEffortProfile, RuntimeDynamicWorkflowConfig, WorkflowTriggerDecision,
96    classify_workflow_trigger, ultracode_reasoning_level_for_model,
97};
98use crate::fake_provider::FakeInferenceEngine;
99use crate::goals::RuntimeGoalController;
100use crate::inference_routing::{
101    InferenceRoutingRequest, RuntimeInferenceRouterConfig, collect_inference_routing_candidates,
102    route_inference_selection, transcript_failure_count_since,
103};
104use crate::instructions::{
105    apply_agent_swarm_mode, apply_codex_multi_agent_mode, apply_model_instruction_overlay,
106    apply_parallel_web_tools, apply_plan_mode, apply_runtime_profile, apply_task_ledger_required,
107    apply_thread_developer_instructions, apply_turn_developer_context,
108};
109use crate::policy_gate::DefaultPolicyGate;
110use crate::reliability::{
111    ReliabilityLimitHit, RuntimeReliabilityConfig, TurnReliabilityState,
112    provider_stream_retry_cause,
113};
114pub use crate::speed_policy::RuntimeSpeedPolicyConfig;
115use crate::speed_policy::{SpeedPolicyState, reasoning_from_decision};
116use crate::subagent_traces::{RuntimeAgentSwarmProgressSink, RuntimeSubagentTraceSink};
117use crate::teams::{TeamManager, TeamMemberStartRequest, TeamStartRequest, TeamState};
118use crate::thread_item_cache::{ThreadItemCache, ThreadItemCacheEntry};
119use crate::tool_result_images::ToolImages;
120use crate::verification_gate::VerificationGateState;
121
122const MAX_TOOL_ROUNDS_PER_TURN: usize = 1024;
123/// Injected when `continue_on_failure_limit` turns a consecutive-tool-failure
124/// limit into a continuation, or as the empty-tool-call persistence nudge, so
125/// the model keeps working instead of ending the turn early.
126const RELIABILITY_CONTINUATION_PROMPT: &str = "Verify the task is fully complete. If any part remains unfinished or unverified, keep working using the available tools — try an alternative approach if one is failing. Only stop once the solution is complete and verified; if it is already complete, restate your final answer.";
127/// Capacity of the runtime event broadcast ring buffer. The TUI drains this on
128/// its render loop, which during an active turn only wakes at the ~6 FPS status
129/// animation cadence (backend events do not wake the input poll), so up to
130/// ~166ms of events buffer between drains. A reasoning-high provider that runs
131/// its whole tool loop inside one turn (e.g. claude-code) streams a firehose of
132/// `InferenceEventReceived` deltas plus per-step tool/thinking events; the old
133/// 1024-slot buffer overflowed in those windows and silently dropped tool and
134/// thinking rows from the live view. Sized for generous headroom across bursts
135/// and brief render stalls.
136const EVENT_BUS_CAPACITY: usize = 16_384;
137
138/// Turn id reported to session-scoped local hooks. `SessionStart` and
139/// `SessionEnd` bracket the session rather than any one turn, but the hook
140/// payload carries a turn id, so they share this synthetic value.
141const SESSION_HOOK_TURN_ID: &str = "session";
142pub(crate) const FINAL_ANSWER_PHASE: &str = "final_answer";
143pub(crate) const TASK_LEDGER_TOOL_NAME: &str = "task_ledger.update";
144const TASK_LEDGER_COMPLETION_REMINDER_LIMIT: u8 = 2;
145const TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS: u64 = 180;
146const TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT: u8 = 1;
147pub(crate) const MIN_CHILD_DEADLINE_SECONDS: u64 = 2;
148const MODEL_PROFILE_TRACE_KIND: &str = "model_profile_segment";
149const MODEL_SWITCH_SUMMARY_PREFIX: &str = "Model switch summary:";
150const PROVIDER_CLEANUP_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5);
151
152#[derive(Clone, Copy, Debug, Eq, PartialEq)]
153enum InferenceTimeoutAction {
154    ScoreableCheckpoint,
155    Finalization,
156}
157
158#[derive(Debug, Clone)]
159pub struct RuntimeConfig {
160    pub default_provider: String,
161    pub default_model: String,
162    pub reasoning: Option<String>,
163    pub auto_compact_token_limit: Option<u32>,
164    pub file_backed_dynamic_context: bool,
165    pub hosted_web_search: HostedWebSearchConfig,
166    pub tool_search: ToolSearchConfig,
167    pub provider_tool_search: HashMap<String, ToolSearchConfigOverlay>,
168    pub model_tool_search: HashMap<String, ToolSearchConfigOverlay>,
169    pub model_edit_tools: HashMap<String, String>,
170    pub model_parallel_tool_calls: HashMap<String, bool>,
171    pub model_profiles: HashMap<String, ModelHarnessProfile>,
172    pub tool_allowlist: Vec<String>,
173    /// Seconds a host-executed external tool call may stay unresolved before it fails with a timeout error.
174    pub external_tool_timeout_seconds: u64,
175    pub command_shell: String,
176    pub workspace: Option<String>,
177    pub policy_mode: PolicyMode,
178    /// Whether agent-swarm mode is active (roadmap 104). When on, the runtime
179    /// injects the swarm reminder into each turn's developer instructions so any
180    /// client benefits, not just the TUI.
181    pub agent_swarm_mode: bool,
182    /// Whether ultra mode is active. When on, the runtime injects proactive
183    /// multi-agent delegation policy for any model (not only Codex Sol/Terra
184    /// Ultra effort). Selecting Ultra reasoning on Sol/Terra still enables
185    /// proactive multi-agent for that model without requiring this flag.
186    pub ultra_mode: bool,
187    pub runtime_profile: RuntimeProfile,
188    pub inference_router: RuntimeInferenceRouterConfig,
189    pub speed_policy: RuntimeSpeedPolicyConfig,
190    pub dynamic_workflows: RuntimeDynamicWorkflowConfig,
191    pub reliability: RuntimeReliabilityConfig,
192    /// Positive wall-clock limit for a turn in any runtime profile. `None` or zero is unbounded.
193    pub turn_deadline_seconds: Option<u64>,
194    pub remote_runner_destination: Option<RunnerDestination>,
195    pub team_data_dir: Option<PathBuf>,
196    pub roadmap_data_dir: Option<PathBuf>,
197    pub media_generation: crate::media_generation::RuntimeMediaGenerationConfig,
198    /// `[review]` settings: the review sub-turn and its publishers.
199    pub review: crate::review::RuntimeReviewConfig,
200}
201
202impl Default for RuntimeConfig {
203    fn default() -> Self {
204        Self {
205            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
206            default_model: "mock".to_string(),
207            reasoning: None,
208            auto_compact_token_limit: None,
209            file_backed_dynamic_context: true,
210            hosted_web_search: HostedWebSearchConfig::cached(),
211            tool_search: ToolSearchConfig::default(),
212            provider_tool_search: HashMap::new(),
213            model_tool_search: HashMap::new(),
214            model_edit_tools: HashMap::new(),
215            model_parallel_tool_calls: HashMap::new(),
216            model_profiles: HashMap::new(),
217            tool_allowlist: Vec::new(),
218            external_tool_timeout_seconds: DEFAULT_EXTERNAL_TOOL_TIMEOUT_SECONDS,
219            command_shell: roder_api::command_shell::default_command_shell(),
220            workspace: None,
221            policy_mode: PolicyMode::Default,
222            agent_swarm_mode: false,
223            ultra_mode: false,
224            runtime_profile: RuntimeProfile::Interactive,
225            inference_router: RuntimeInferenceRouterConfig::default(),
226            speed_policy: RuntimeSpeedPolicyConfig::default(),
227            dynamic_workflows: RuntimeDynamicWorkflowConfig::default(),
228            reliability: RuntimeReliabilityConfig::default(),
229            turn_deadline_seconds: None,
230            remote_runner_destination: None,
231            team_data_dir: None,
232            roadmap_data_dir: None,
233            media_generation: crate::media_generation::RuntimeMediaGenerationConfig::default(),
234            review: crate::review::RuntimeReviewConfig::default(),
235        }
236    }
237}
238
239#[derive(Debug, Clone)]
240pub struct StartTurnRequest {
241    pub thread_id: ThreadId,
242    pub message: String,
243    pub images: Vec<InputImage>,
244    pub provider_override: Option<String>,
245    pub model_override: Option<String>,
246    pub reasoning_override: Option<String>,
247    pub workspace: String,
248    pub instructions: InstructionBundle,
249    /**
250     * Per-turn developer-authority context for this turn's InstructionBundle.
251     * Applies to every inference round of the turn, is never written to
252     * thread state. Automatic goal continuations retain it; later explicit
253     * turns receive only the context supplied by their host.
254     */
255    pub developer_context: Option<String>,
256    pub task_ledger_required: bool,
257    /**
258     * Per-turn provider service tier (OpenAI `service_tier`, e.g.
259     * `"priority"` for Fast mode). Carried to every inference round of the
260     * turn as `RuntimeHints::service_tier`; `None` inherits the thread and provider
261     * default. Providers without a tier concept ignore it.
262     */
263    pub service_tier_override: Option<String>,
264}
265
266#[derive(Debug, Clone)]
267pub struct CreateThreadRequest {
268    /// Default provider service tier for this thread.
269    pub service_tier: Option<String>,
270    pub title: Option<String>,
271    pub workspace: String,
272    pub workspace_id: Option<String>,
273    pub root_id: Option<String>,
274    pub provider: Option<String>,
275    pub model: Option<String>,
276    pub selection_mode: Option<ModelSelectionMode>,
277    /// Per-thread tool filter applied on top of the runtime allowlist. Empty = no filtering.
278    pub tool_allowlist: Vec<String>,
279    /// Host-supplied instructions added to the developer slot of every turn's inference request.
280    pub developer_instructions: Option<String>,
281    /// Host-executed tool specs advertised to the model on every turn of this thread.
282    pub external_tools: Vec<roder_api::tools::ToolSpec>,
283    /// Explicit remote-runner binding for the thread's native coding tools.
284    pub runner: Option<ThreadRunnerSelection>,
285}
286
287/**
288 * Thread-level remote-runner selection. The destination config is persisted
289 * with the thread, so secrets must reach the provider through its
290 * environment, not this config.
291 */
292#[derive(Debug, Clone)]
293pub struct ThreadRunnerSelection {
294    pub provider_id: String,
295    pub config: serde_json::Value,
296    /// Absolute path on the runner used as the thread's coding-tool workspace root.
297    pub workspace: String,
298    /**
299     * Extra absolute runner paths file reads may resolve under, beyond
300     * `workspace`. Writes and the working directory stay confined to
301     * `workspace`.
302     */
303    pub read_roots: Vec<String>,
304}
305
306#[derive(Debug, Clone, PartialEq, Eq)]
307pub struct PendingPlanExit {
308    pub thread_id: ThreadId,
309    pub turn_id: TurnId,
310    pub request_id: String,
311    pub target_mode: PolicyMode,
312    pub plan_summary: Option<String>,
313    pub next_steps: Vec<String>,
314    pub requested_at: OffsetDateTime,
315    pub expires_at: Option<OffsetDateTime>,
316}
317
318pub(crate) struct PendingToolApproval {
319    pub(crate) thread_id: ThreadId,
320    pub(crate) turn_id: TurnId,
321    pub(crate) tool_id: String,
322    pub(crate) tool_name: String,
323    pub(crate) call: roder_api::tools::ToolCall,
324    pub(crate) context: Option<ToolExecutionContext>,
325    pub(crate) tx: oneshot::Sender<bool>,
326}
327
328pub(crate) struct PendingUserInput {
329    pub(crate) thread_id: ThreadId,
330    pub(crate) turn_id: TurnId,
331    pub(crate) tx: oneshot::Sender<serde_json::Value>,
332}
333
334/// Host answer to an external tool call delivered via `tools/resolve`.
335#[derive(Debug, Clone, PartialEq, Eq)]
336pub struct ExternalToolResolution {
337    pub output: String,
338    pub is_error: bool,
339}
340
341pub(crate) struct PendingExternalToolCall {
342    pub(crate) thread_id: ThreadId,
343    pub(crate) turn_id: TurnId,
344    pub(crate) tool_id: String,
345    pub(crate) tool_name: String,
346    pub(crate) tx: oneshot::Sender<ExternalToolResolution>,
347}
348
349#[derive(Clone)]
350struct ActiveTurnHandle {
351    thread_id: ThreadId,
352    abort: AbortHandle,
353    steers: Arc<Mutex<Vec<QueuedTurnSteer>>>,
354    steer_changed: tokio::sync::watch::Sender<u64>,
355    drain: Arc<TurnDrainHandle>,
356}
357
358/// Tracks a task after it has been interrupted and removed from the interactive
359/// active-turn map. This lets users start a follow-up turn immediately while a
360/// bounded runtime shutdown can still await the original task's cleanup.
361struct TurnDrainHandle {
362    thread_id: ThreadId,
363    interrupt_requested: AtomicBool,
364    interrupt_reason: Mutex<Option<TurnLifecycleReason>>,
365    completed: AtomicBool,
366    completed_notify: Notify,
367}
368
369/// Result of a bounded runtime drain. `Clean` proves the runtime's own turn
370/// tasks have reached their cleanup paths; provider-specific child-process
371/// reaping remains an explicit provider capability.
372#[derive(Debug, Clone, PartialEq, Eq)]
373pub enum RuntimeDrainOutcome {
374    Clean {
375        interrupted_turn_ids: Vec<TurnId>,
376    },
377    DeadlineExceeded {
378        interrupted_turn_ids: Vec<TurnId>,
379        remaining_turn_ids: Vec<TurnId>,
380    },
381    /// At least one lifecycle transition initiated by this drain could not be
382    /// durably appended. The runtime still made its best cleanup attempt, but
383    /// callers must not claim a persisted terminal state as proof of recovery.
384    PersistenceFailed {
385        interrupted_turn_ids: Vec<TurnId>,
386        remaining_turn_ids: Vec<TurnId>,
387    },
388}
389
390#[derive(Clone)]
391struct QueuedTurnSteer {
392    message: UserMessage,
393    mailbox_ack: Option<MailboxDeliveryAck>,
394}
395
396#[derive(Clone)]
397struct MailboxDeliveryAck {
398    team_id: TeamId,
399    message_ids: Vec<String>,
400}
401
402#[derive(Clone)]
403struct InheritedTurnContext {
404    workspace: String,
405    instructions: InstructionBundle,
406    developer_context: Option<String>,
407}
408
409#[derive(Debug, Clone, Default, PartialEq, Eq)]
410pub struct ThreadActivity {
411    pub active_turn_id: Option<TurnId>,
412    pub active_flags: Vec<String>,
413}
414
415/// Per-thread settings persisted at thread creation and applied to every turn.
416#[derive(Debug, Clone, Default)]
417pub(crate) struct ThreadTurnOverrides {
418    pub(crate) tool_allowlist: Vec<String>,
419    pub(crate) developer_instructions: Option<String>,
420    pub(crate) external_tools: Vec<roder_api::tools::ToolSpec>,
421    pub(crate) service_tier: Option<String>,
422}
423
424#[derive(Debug, Clone, Copy, PartialEq, Eq)]
425pub(crate) enum TurnRunOutcome {
426    Completed,
427    Stopped,
428}
429
430impl PendingPlanExit {
431    pub fn new(
432        thread_id: ThreadId,
433        turn_id: TurnId,
434        request_id: String,
435        target_mode: PolicyMode,
436        plan_summary: Option<String>,
437        next_steps: Vec<String>,
438    ) -> Self {
439        let requested_at = OffsetDateTime::now_utc();
440        Self {
441            thread_id,
442            turn_id,
443            request_id,
444            target_mode,
445            plan_summary,
446            next_steps,
447            requested_at,
448            expires_at: Some(requested_at + default_plan_exit_timeout()),
449        }
450    }
451
452    pub fn is_expired(&self, now: OffsetDateTime) -> bool {
453        self.expires_at.is_some_and(|expires_at| now >= expires_at)
454    }
455}
456
457pub fn default_plan_exit_timeout() -> Duration {
458    Duration::minutes(10)
459}
460
461pub const DEFAULT_EXTERNAL_TOOL_TIMEOUT_SECONDS: u64 = 300;
462
463fn format_mailbox_messages(team: &TeamState, messages: &[TeamMailboxMessage]) -> String {
464    messages
465        .iter()
466        .map(|message| {
467            let recipient = team
468                .members
469                .iter()
470                .find(|member| member.id == message.to_member_id)
471                .map(canonical_team_member_path)
472                .unwrap_or_else(|| format!("/root/{}", message.to_member_id));
473            let sender = message
474                .from_member_id
475                .as_deref()
476                .and_then(|id| team.members.iter().find(|member| member.id == id))
477                .map(canonical_team_member_path)
478                .unwrap_or_else(|| "/root".to_string());
479            let message_type = match message.kind {
480                TeamMailboxMessageKind::Message => "MESSAGE",
481                TeamMailboxMessageKind::NewTask => "NEW_TASK",
482                TeamMailboxMessageKind::FinalAnswer => "FINAL_ANSWER",
483            };
484            format!(
485                "Message Type: {message_type}\nTask name: {recipient}\nSender: {sender}\nPayload:\n{}",
486                message.text
487            )
488        })
489        .collect::<Vec<_>>()
490        .join("\n\n")
491}
492
493fn canonical_team_member_path(member: &TeamMemberDescriptor) -> String {
494    if let Some(agent_path) = member
495        .agent_path
496        .as_deref()
497        .filter(|path| *path == "/root" || path.starts_with("/root/"))
498    {
499        return agent_path.to_string();
500    }
501    if member.role == TeamMemberRole::Lead {
502        return "/root".to_string();
503    }
504    member
505        .task_name
506        .as_deref()
507        .filter(|name| !name.trim().is_empty())
508        .map(|name| format!("/root/{}", name.trim().trim_matches('/')))
509        .unwrap_or_else(|| format!("/root/{}", member.id))
510}
511
512fn is_codex_v2_team(team: &TeamState) -> bool {
513    team.members.iter().any(|member| {
514        member
515            .model
516            .as_deref()
517            .is_some_and(|model| model_supports_reasoning_effort(model, REASONING_ULTRA))
518    })
519}
520
521fn runtime_local_team_view(
522    mut team: TeamState,
523    active_thread_ids: &std::collections::HashSet<ThreadId>,
524) -> TeamState {
525    for member in &mut team.members {
526        if member.status == TeamMemberStatus::Running
527            && !active_thread_ids.contains(&member.thread_id)
528        {
529            member.status = TeamMemberStatus::Interrupted;
530            member.current_turn_id = None;
531        }
532    }
533    team
534}
535
536type RunnerToolExecutionKey = (String, String);
537type RunnerToolExecutionMutex = Mutex<()>;
538type RunnerToolExecutionLockRegistry =
539    Mutex<HashMap<RunnerToolExecutionKey, Weak<RunnerToolExecutionMutex>>>;
540
541pub struct Runtime {
542    pub bus: EventBus,
543    pub registry: ExtensionRegistry,
544    config: RwLock<RuntimeConfig>,
545    pub(crate) execution_lease: Option<Arc<crate::execution_lease::RuntimeExecutionLease>>,
546    pending_plan_exit: RwLock<Option<PendingPlanExit>>,
547    pub(crate) pending_tool_approvals: Mutex<HashMap<String, PendingToolApproval>>,
548    pub(crate) pending_user_inputs: Mutex<HashMap<String, PendingUserInput>>,
549    pub(crate) pending_external_tool_calls: Mutex<HashMap<String, PendingExternalToolCall>>,
550    /// Serializes turn admission with shutdown quiescing. A drain takes this
551    /// gate before flipping `accepting_turns` and snapshotting active work so a
552    /// turn that observed the old flag cannot be inserted after the snapshot.
553    turn_admission: Mutex<()>,
554    active_turns: RwLock<HashMap<TurnId, ActiveTurnHandle>>,
555    turn_drains: RwLock<HashMap<TurnId, Arc<TurnDrainHandle>>>,
556    turn_usage: Mutex<HashMap<TurnId, Arc<Mutex<usage_accounting::TurnUsageAccounting>>>>,
557    thread_usage_metadata_lock: Mutex<()>,
558    /// Provider cleanup handles register synchronously through the inference
559    /// tool-executor context. They are intentionally separate from active turn
560    /// admission so an interrupted turn can remain drainable after a follow-up
561    /// turn starts on the same thread.
562    provider_turn_cleanups: std::sync::Mutex<HashMap<TurnId, Arc<dyn ProviderTurnCleanup>>>,
563    accepting_turns: AtomicBool,
564    /// Monotonic counter used by bounded drains to surface lifecycle-state
565    /// persistence failures without making an individual provider turn fail.
566    lifecycle_persistence_failures: AtomicUsize,
567    lifecycle_shutdown_drains: AtomicUsize,
568    lifecycle_clean_shutdowns: AtomicUsize,
569    lifecycle_deadline_exceeded: AtomicUsize,
570    lifecycle_persistence_failed_drains: AtomicUsize,
571    lifecycle_restart_reconciliations: AtomicUsize,
572    lifecycle_shutdown_drain_duration_ms_total: AtomicUsize,
573    provider_cleanup_confirmed: AtomicUsize,
574    provider_cleanup_timed_out: AtomicUsize,
575    provider_cleanup_unknown: AtomicUsize,
576    active_turns_changed: Notify,
577    active_turn_selections: RwLock<HashMap<TurnId, ModelSelectionMode>>,
578    compaction_templates: RwLock<HashMap<ThreadId, AgentInferenceRequest>>,
579    thread_admission_gates: Mutex<HashMap<ThreadId, Arc<Mutex<()>>>>,
580    /// Threads that have already dispatched a `SessionStart` local hook.
581    ///
582    /// `ThreadCreated` fires once, but `ThreadLoaded` fires on every read of the
583    /// thread from its store, so without this a session ran its setup hooks
584    /// several times — including after the turn had already stopped.
585    session_hook_started: RwLock<HashSet<ThreadId>>,
586    active_turn_contexts: RwLock<HashMap<TurnId, InheritedTurnContext>>,
587    /// Process-local execution boundary. Local/CLI runtimes default to true;
588    /// hosted runtime pools set this false so missing runner metadata fails
589    /// closed instead of exposing the host workspace.
590    allow_local_workspaces: AtomicBool,
591    // Spawn-time live authority retained for every reusable turn of a long-lived teammate.
592    // This stays process-local because developer_context is explicitly volatile and must never
593    // be persisted to thread state.
594    team_member_turn_contexts: Mutex<HashMap<ThreadId, InheritedTurnContext>>,
595    workspace: PathBuf,
596    pub(crate) teams: TeamManager,
597    agent_team_spawn_lock: Mutex<()>,
598    pub(crate) roadmaps: Mutex<roder_roadmap::RoadmapRuntime>,
599    /// Completed reviews, most recent last, so a later publish can resolve a
600    /// review id back to its findings. Bounded; see `review::run`.
601    pub(crate) reviews: Mutex<Vec<crate::review::ReviewRecord>>,
602    pub(crate) goals: Arc<RuntimeGoalController>,
603    context_artifacts: roder_api::artifacts::ContextArtifactStore,
604    pub(crate) thread_store: Option<Arc<dyn ThreadStore>>,
605    thread_item_cache: Mutex<ThreadItemCache>,
606    pub(crate) tool_registry: ToolRegistry,
607    media_generation: Arc<crate::media_generation::MediaGenerationService>,
608    pub(crate) skills: RwLock<SkillRegistry>,
609    /// Lazily-started bounded dispatch of emitted events to registry
610    /// `EventSink`s (process extensions etc.); see `event_sink_dispatch`.
611    event_sink_dispatcher: tokio::sync::OnceCell<crate::event_sink_dispatch::EventSinkDispatcher>,
612    pub(crate) compaction_hysteresis:
613        std::sync::Mutex<HashMap<ThreadId, crate::compaction_runtime::CompactionState>>,
614    /// Per-thread agent-swarm mode overrides (roadmap 104). A thread present in
615    /// this map uses its stored value; absent threads fall back to the
616    /// runtime-global `RuntimeConfig.agent_swarm_mode` default. This mirrors the
617    /// team per-member policy-mode override idiom so swarm mode is per-thread
618    /// like the other `thread/*` operations, without a separate runtime.
619    agent_swarm_modes: RwLock<HashMap<ThreadId, bool>>,
620    /// Per-thread ultra mode overrides. A thread present in this map uses its
621    /// stored value; absent threads fall back to the runtime-global
622    /// `RuntimeConfig.ultra_mode` default.
623    ultra_modes: RwLock<HashMap<ThreadId, bool>>,
624    /**
625     * Live remote-runner sessions keyed by thread. Each per-thread mutex is
626     * held across provider initialization, making the first workspace-tool
627     * access a singleflight while allowing unrelated threads to initialize in
628     * parallel. Failed initialization is shared with current waiters, then
629     * evicted so a later tool call can retry.
630     */
631    runner_sessions: Arc<Mutex<HashMap<ThreadId, Arc<RunnerSessionSlot>>>>,
632    /**
633     * Outer tool-call locks keyed by the actual remote session, not the
634     * per-thread destination id. Hosted runtimes are tenant-scoped, so this
635     * serializes threads that share one sandbox without coupling different
636     * tenants or independent runner sessions.
637     */
638    runner_tool_execution_locks: RunnerToolExecutionLockRegistry,
639}
640
641#[derive(Clone)]
642struct CachedRunnerSession {
643    destination: RunnerDestination,
644    session: Arc<dyn RemoteRunnerSession>,
645}
646
647struct RunnerSessionSlot {
648    provider_id: String,
649    destination_id: String,
650    state: Mutex<RunnerSessionSlotState>,
651    initialized: Notify,
652}
653
654enum RunnerSessionSlotState {
655    Empty,
656    Initializing,
657    Ready(CachedRunnerSession),
658    Failed(Arc<str>),
659}
660
661impl RunnerSessionSlot {
662    fn empty(destination: &RunnerDestination) -> Self {
663        Self {
664            provider_id: destination.provider_id.clone(),
665            destination_id: destination.id.clone(),
666            state: Mutex::new(RunnerSessionSlotState::Empty),
667            initialized: Notify::new(),
668        }
669    }
670
671    fn ready(session: CachedRunnerSession) -> Self {
672        Self {
673            provider_id: session.destination.provider_id.clone(),
674            destination_id: session.destination.id.clone(),
675            state: Mutex::new(RunnerSessionSlotState::Ready(session)),
676            initialized: Notify::new(),
677        }
678    }
679
680    fn matches(&self, destination: &RunnerDestination) -> bool {
681        self.provider_id == destination.provider_id && self.destination_id == destination.id
682    }
683}
684
685impl Runtime {
686    pub fn new(registry: ExtensionRegistry, config: RuntimeConfig) -> anyhow::Result<Self> {
687        if registry.inference_engines.is_empty() {
688            anyhow::bail!("at least one inference engine must be registered");
689        }
690        validate_runtime_config_reasoning(&config)?;
691        validate_runtime_inference_router_config(&registry, &config)?;
692
693        let bus = EventBus::new(EVENT_BUS_CAPACITY);
694        let thread_store = registry
695            .thread_stores
696            .first()
697            .map(|factory| factory.create());
698        let mut tool_registry = ToolRegistry::default();
699        for contributor in &registry.tools {
700            contributor
701                .contribute(&mut tool_registry)
702                .with_context(|| format!("tool contributor {} failed", contributor.id()))?;
703        }
704        crate::agent_control_tools::contribute_agent_control_tools(&mut tool_registry)?;
705
706        let media_generation = Arc::new(crate::media_generation::MediaGenerationService::new(
707            registry.media_generator_providers.clone(),
708            config.media_generation.clone(),
709        ));
710        tool_registry.replace(Arc::new(
711            crate::media_generation::MediaGenerateImageTool::new(media_generation.clone()),
712        ));
713
714        let team_data_dir = config.team_data_dir.clone();
715        let workspace = config
716            .workspace
717            .clone()
718            .map(PathBuf::from)
719            .unwrap_or(std::env::current_dir()?);
720        let roadmap_data_dir = config
721            .roadmap_data_dir
722            .clone()
723            .unwrap_or_else(|| workspace.join(".roder"));
724        let context_artifacts = thread_store
725            .as_ref()
726            .and_then(|store| store.context_artifact_store())
727            .or_else(|| {
728                thread_store
729                    .as_ref()
730                    .and_then(|store| store.local_thread_root())
731                    .map(FilesystemContextArtifactStore::shared_thread_scoped)
732            })
733            .unwrap_or_else(|| {
734                FilesystemContextArtifactStore::shared_legacy(default_context_artifact_dir())
735            });
736        let goals = Arc::new(RuntimeGoalController::new(
737            bus.clone(),
738            thread_store.clone(),
739        ));
740        let runtime = Self {
741            bus,
742            registry,
743            config: RwLock::new(config),
744            execution_lease: None,
745            pending_plan_exit: RwLock::new(None),
746            pending_tool_approvals: Mutex::new(HashMap::new()),
747            pending_user_inputs: Mutex::new(HashMap::new()),
748            pending_external_tool_calls: Mutex::new(HashMap::new()),
749            turn_admission: Mutex::new(()),
750            active_turns: RwLock::new(HashMap::new()),
751            turn_drains: RwLock::new(HashMap::new()),
752            turn_usage: Mutex::new(HashMap::new()),
753            thread_usage_metadata_lock: Mutex::new(()),
754            provider_turn_cleanups: std::sync::Mutex::new(HashMap::new()),
755            accepting_turns: AtomicBool::new(true),
756            lifecycle_persistence_failures: AtomicUsize::new(0),
757            lifecycle_shutdown_drains: AtomicUsize::new(0),
758            lifecycle_clean_shutdowns: AtomicUsize::new(0),
759            lifecycle_deadline_exceeded: AtomicUsize::new(0),
760            lifecycle_persistence_failed_drains: AtomicUsize::new(0),
761            lifecycle_restart_reconciliations: AtomicUsize::new(0),
762            lifecycle_shutdown_drain_duration_ms_total: AtomicUsize::new(0),
763            provider_cleanup_confirmed: AtomicUsize::new(0),
764            provider_cleanup_timed_out: AtomicUsize::new(0),
765            provider_cleanup_unknown: AtomicUsize::new(0),
766            active_turns_changed: Notify::new(),
767            active_turn_selections: RwLock::new(HashMap::new()),
768            compaction_templates: RwLock::new(HashMap::new()),
769            thread_admission_gates: Mutex::new(HashMap::new()),
770            session_hook_started: RwLock::new(HashSet::new()),
771            active_turn_contexts: RwLock::new(HashMap::new()),
772            allow_local_workspaces: AtomicBool::new(true),
773            team_member_turn_contexts: Mutex::new(HashMap::new()),
774            workspace: workspace.clone(),
775            teams: TeamManager::new(
776                team_data_dir.unwrap_or_else(crate::teams::default_team_data_dir),
777            ),
778            agent_team_spawn_lock: Mutex::new(()),
779            roadmaps: Mutex::new(roder_roadmap::RoadmapRuntime::new(
780                workspace,
781                roadmap_data_dir,
782            )),
783            reviews: Mutex::new(Vec::new()),
784            goals,
785            context_artifacts,
786            thread_store,
787            thread_item_cache: Mutex::new(ThreadItemCache::default()),
788            tool_registry,
789            media_generation,
790            skills: RwLock::new(SkillRegistry::load(SkillRegistryOptions::new(
791                PathBuf::new(),
792            ))),
793            event_sink_dispatcher: tokio::sync::OnceCell::new(),
794            compaction_hysteresis: crate::compaction_runtime::compaction_hysteresis_state(),
795            agent_swarm_modes: RwLock::new(HashMap::new()),
796            ultra_modes: RwLock::new(HashMap::new()),
797            runner_sessions: Arc::new(Mutex::new(HashMap::new())),
798            runner_tool_execution_locks: Mutex::new(HashMap::new()),
799        };
800        runtime.bus.emit(RoderEvent::RuntimeStarted(RuntimeStarted {
801            timestamp: OffsetDateTime::now_utc(),
802        }));
803        for manifest in &runtime.registry.manifests {
804            runtime
805                .bus
806                .emit(RoderEvent::ExtensionRegistered(ExtensionRegistered {
807                    extension_id: manifest.id.clone(),
808                    timestamp: OffsetDateTime::now_utc(),
809                }));
810        }
811        Ok(runtime)
812    }
813
814    pub fn from_engine(engine: Arc<dyn InferenceEngine>) -> anyhow::Result<Self> {
815        let mut builder = roder_api::extension::ExtensionRegistryBuilder::new();
816        builder.inference_engine(engine);
817        Self::new(builder.build()?, RuntimeConfig::default())
818    }
819
820    pub fn fake() -> anyhow::Result<Self> {
821        Self::from_engine(Arc::new(FakeInferenceEngine))
822    }
823
824    pub fn subscribe_events(&self) -> tokio::sync::broadcast::Receiver<EventEnvelope> {
825        self.bus.subscribe()
826    }
827
828    /// Returns process-local, redacted lifecycle health counters. The snapshot
829    /// intentionally has fixed fields and contains no provider, command,
830    /// process, thread, or turn identifiers.
831    pub fn lifecycle_metrics(&self) -> LifecycleMetricsSnapshot {
832        LifecycleMetricsSnapshot {
833            shutdown_drain_count: self.lifecycle_shutdown_drains.load(Ordering::Acquire) as u64,
834            clean_shutdown_count: self.lifecycle_clean_shutdowns.load(Ordering::Acquire) as u64,
835            deadline_exceeded_count: self.lifecycle_deadline_exceeded.load(Ordering::Acquire)
836                as u64,
837            persistence_failed_count: self
838                .lifecycle_persistence_failed_drains
839                .load(Ordering::Acquire) as u64,
840            restart_reconciliation_count: self
841                .lifecycle_restart_reconciliations
842                .load(Ordering::Acquire) as u64,
843            lifecycle_persistence_failure_count: self
844                .lifecycle_persistence_failures
845                .load(Ordering::Acquire) as u64,
846            shutdown_drain_duration_ms_total: self
847                .lifecycle_shutdown_drain_duration_ms_total
848                .load(Ordering::Acquire) as u64,
849            provider_cleanup_confirmed_count: self
850                .provider_cleanup_confirmed
851                .load(Ordering::Acquire) as u64,
852            provider_cleanup_timed_out_count: self
853                .provider_cleanup_timed_out
854                .load(Ordering::Acquire) as u64,
855            provider_cleanup_unknown_count: self.provider_cleanup_unknown.load(Ordering::Acquire)
856                as u64,
857        }
858    }
859
860    async fn persist_turn_lifecycle_record(&self, record: &TurnLifecycleRecord) -> bool {
861        let Some(store) = &self.thread_store else {
862            return true;
863        };
864        let state = match record.extension_state() {
865            Ok(state) => state,
866            Err(_) => {
867                self.lifecycle_persistence_failures
868                    .fetch_add(1, Ordering::AcqRel);
869                return false;
870            }
871        };
872        // Lifecycle diagnostics must not turn a provider result into a failed
873        // turn merely because a third-party store lacks extension-state support.
874        // JSONL/Postgres stores persist this append-only record atomically at
875        // their own storage boundary.
876        if store
877            .append_extension_state(&record.thread_id, &state)
878            .await
879            .is_err()
880        {
881            self.lifecycle_persistence_failures
882                .fetch_add(1, Ordering::AcqRel);
883            return false;
884        }
885        true
886    }
887
888    async fn record_turn_lifecycle(
889        &self,
890        thread_id: ThreadId,
891        turn_id: TurnId,
892        state: TurnLifecycleState,
893        cleanup: TurnCleanupState,
894        reason: Option<TurnLifecycleReason>,
895    ) -> TurnLifecycleRecord {
896        self.record_turn_lifecycle_with_ownership(
897            thread_id,
898            turn_id,
899            state,
900            cleanup,
901            reason,
902            TurnCleanupOwnership::RuntimeTaskOnly,
903        )
904        .await
905    }
906
907    async fn record_turn_lifecycle_with_ownership(
908        &self,
909        thread_id: ThreadId,
910        turn_id: TurnId,
911        state: TurnLifecycleState,
912        cleanup: TurnCleanupState,
913        reason: Option<TurnLifecycleReason>,
914        ownership: TurnCleanupOwnership,
915    ) -> TurnLifecycleRecord {
916        let record = TurnLifecycleRecord::new(
917            thread_id,
918            turn_id,
919            state,
920            cleanup,
921            reason,
922            OffsetDateTime::now_utc(),
923        )
924        .with_ownership(ownership);
925        let _ = self.persist_turn_lifecycle_record(&record).await;
926        self.emit(RoderEvent::TurnLifecycleUpdated(record.clone()))
927            .await;
928        record
929    }
930
931    pub(crate) fn register_provider_turn_cleanup(
932        &self,
933        turn_id: &TurnId,
934        cleanup: Arc<dyn ProviderTurnCleanup>,
935    ) {
936        if let Ok(mut cleanups) = self.provider_turn_cleanups.lock() {
937            cleanups.insert(turn_id.clone(), cleanup);
938        }
939    }
940
941    fn provider_cleanup_ownership(&self, turn_id: &TurnId) -> TurnCleanupOwnership {
942        self.provider_turn_cleanups
943            .lock()
944            .ok()
945            .and_then(|cleanups| cleanups.get(turn_id).cloned())
946            .map(|cleanup| cleanup.ownership())
947            .unwrap_or(TurnCleanupOwnership::RuntimeTaskOnly)
948    }
949
950    async fn await_provider_turn_cleanup(
951        &self,
952        turn_id: &TurnId,
953    ) -> (TurnCleanupState, TurnCleanupOwnership) {
954        let cleanup = self
955            .provider_turn_cleanups
956            .lock()
957            .ok()
958            .and_then(|mut cleanups| cleanups.remove(turn_id));
959        let Some(cleanup) = cleanup else {
960            return (
961                TurnCleanupState::Unknown,
962                TurnCleanupOwnership::RuntimeTaskOnly,
963            );
964        };
965        let ownership = cleanup.ownership();
966        match tokio::time::timeout(PROVIDER_CLEANUP_TIMEOUT, cleanup.wait_for_cleanup()).await {
967            Ok(Ok(())) => {
968                self.provider_cleanup_confirmed
969                    .fetch_add(1, Ordering::AcqRel);
970                (
971                    TurnCleanupState::Completed,
972                    TurnCleanupOwnership::ProviderCleanupConfirmed,
973                )
974            }
975            Ok(Err(_)) => {
976                self.provider_cleanup_unknown.fetch_add(1, Ordering::AcqRel);
977                (TurnCleanupState::Unknown, ownership)
978            }
979            Err(_) => {
980                self.provider_cleanup_timed_out
981                    .fetch_add(1, Ordering::AcqRel);
982                (TurnCleanupState::TimedOut, ownership)
983            }
984        }
985    }
986
987    fn record_lifecycle_drain_outcome(
988        &self,
989        started_at: tokio::time::Instant,
990        outcome: &RuntimeDrainOutcome,
991    ) {
992        self.lifecycle_shutdown_drains
993            .fetch_add(1, Ordering::AcqRel);
994        self.lifecycle_shutdown_drain_duration_ms_total.fetch_add(
995            started_at.elapsed().as_millis().min(usize::MAX as u128) as usize,
996            Ordering::AcqRel,
997        );
998        match outcome {
999            RuntimeDrainOutcome::Clean { .. } => {
1000                self.lifecycle_clean_shutdowns
1001                    .fetch_add(1, Ordering::AcqRel);
1002            }
1003            RuntimeDrainOutcome::DeadlineExceeded { .. } => {
1004                self.lifecycle_deadline_exceeded
1005                    .fetch_add(1, Ordering::AcqRel);
1006            }
1007            RuntimeDrainOutcome::PersistenceFailed { .. } => {
1008                self.lifecycle_persistence_failed_drains
1009                    .fetch_add(1, Ordering::AcqRel);
1010            }
1011        }
1012    }
1013
1014    fn lifecycle_record_for_event(event: &RoderEvent) -> Option<TurnLifecycleRecord> {
1015        match event {
1016            RoderEvent::TurnStarted(event) => Some(TurnLifecycleRecord::new(
1017                event.thread_id.clone(),
1018                event.turn_id.clone(),
1019                TurnLifecycleState::Running,
1020                TurnCleanupState::NotRequested,
1021                None,
1022                event.timestamp,
1023            )),
1024            // `run_turn` persists completed turns after it has awaited a
1025            // registered provider cleanup handle. Do not let the generic event
1026            // projection overwrite that acknowledgement with an unproven state.
1027            RoderEvent::TurnCompleted(_) => None,
1028            // Some failure paths already have an event before the turn wrapper
1029            // reaches its cleanup acknowledgement. Preserve a durable fallback
1030            // reason here; the wrapper appends a later record with the more
1031            // precise cleanup outcome when a provider registered one.
1032            RoderEvent::TurnFailed(event) => Some(TurnLifecycleRecord::new(
1033                event.thread_id.clone(),
1034                event.turn_id.clone(),
1035                TurnLifecycleState::Failed,
1036                TurnCleanupState::Unknown,
1037                Some(if event.error_kind.as_deref() == Some("deadline_timeout") {
1038                    TurnLifecycleReason::DeadlineExceeded
1039                } else {
1040                    TurnLifecycleReason::ProviderFailure
1041                }),
1042                event.timestamp,
1043            )),
1044            RoderEvent::TurnDeadlineExceeded(_) | RoderEvent::TurnInterrupted(_) => None,
1045            RoderEvent::TurnLifecycleUpdated(_) => None,
1046            _ => None,
1047        }
1048    }
1049
1050    pub fn registry(&self) -> &ExtensionRegistry {
1051        &self.registry
1052    }
1053
1054    pub fn media_generation(&self) -> Arc<crate::media_generation::MediaGenerationService> {
1055        self.media_generation.clone()
1056    }
1057
1058    pub fn context_artifacts(&self) -> roder_api::artifacts::ContextArtifactStore {
1059        self.context_artifacts.clone()
1060    }
1061
1062    pub async fn execute_workflow_tool(
1063        &self,
1064        thread_id: ThreadId,
1065        tool_name: &str,
1066        arguments: serde_json::Value,
1067    ) -> anyhow::Result<ToolResult> {
1068        let Some(executor) = self.tool_registry.get(tool_name) else {
1069            anyhow::bail!("tool not found: {tool_name}");
1070        };
1071        let tool_call = ToolCall {
1072            id: format!("slash-{tool_name}"),
1073            name: tool_name.to_string(),
1074            raw_arguments: serde_json::to_string(&arguments)?,
1075            arguments,
1076            thread_id: thread_id.clone(),
1077            turn_id: "slash-command".to_string(),
1078        };
1079        let runtime_config = self.status().await;
1080        let ctx = self.tool_execution_context(
1081            thread_id,
1082            "slash-command".to_string(),
1083            runtime_config.policy_mode,
1084            runtime_config.workspace.as_deref(),
1085            Some(&runtime_config.command_shell),
1086        );
1087        executor.execute(ctx, tool_call).await
1088    }
1089
1090    pub(crate) fn tool_execution_context(
1091        &self,
1092        thread_id: ThreadId,
1093        turn_id: TurnId,
1094        mode: PolicyMode,
1095        workspace: Option<&str>,
1096        command_shell: Option<&str>,
1097    ) -> ToolExecutionContext {
1098        let mut ctx = ToolExecutionContext::new(thread_id, turn_id, mode)
1099            .with_command_shell(command_shell.unwrap_or_default())
1100            .with_process_runner(Arc::new(LocalProcessRunner))
1101            .with_context_artifacts(self.context_artifacts.backend())
1102            .with_goal_controller(self.goals.clone())
1103            .with_subagent_trace_sink(Arc::new(RuntimeSubagentTraceSink::new(
1104                self.bus.clone(),
1105                self.thread_store.clone(),
1106            )))
1107            .with_swarm_progress_sink(Arc::new(RuntimeAgentSwarmProgressSink::new(
1108                self.bus.clone(),
1109                self.thread_store.clone(),
1110            )));
1111        if let Some(workspace) = workspace {
1112            ctx = ctx.with_workspace_handle(Arc::new(ScopedFilesystem::new(workspace)));
1113        }
1114        ctx
1115    }
1116
1117    pub async fn status(&self) -> RuntimeConfig {
1118        self.config.read().await.clone()
1119    }
1120
1121    pub async fn set_skills(&self, skills: SkillRegistry) {
1122        *self.skills.write().await = skills;
1123    }
1124
1125    pub async fn skills_snapshot(&self) -> SkillRegistry {
1126        self.skills.read().await.clone()
1127    }
1128
1129    pub fn workspace(&self) -> PathBuf {
1130        self.workspace.clone()
1131    }
1132
1133    /// Controls whether native workspace tools may execute without an
1134    /// explicit remote-runner binding. Hosted runtimes disable this after
1135    /// construction; ordinary local runtimes retain the compatible default.
1136    pub fn set_allow_local_workspaces(&self, allowed: bool) {
1137        self.allow_local_workspaces
1138            .store(allowed, Ordering::Relaxed);
1139    }
1140
1141    pub fn allows_local_workspaces(&self) -> bool {
1142        self.allow_local_workspaces.load(Ordering::Relaxed)
1143    }
1144
1145    pub async fn set_remote_runner_destination(&self, destination: Option<RunnerDestination>) {
1146        let lifecycle = destination.as_ref().map(|destination| RunnerLifecycle {
1147            destination_id: destination.id.clone(),
1148            provider_id: destination.provider_id.clone(),
1149            state: "configured".to_string(),
1150            session_id: None,
1151            timestamp: OffsetDateTime::now_utc(),
1152        });
1153        self.config.write().await.remote_runner_destination = destination;
1154        if let Some(lifecycle) = lifecycle {
1155            self.emit(RoderEvent::RunnerLifecycle(lifecycle)).await;
1156        } else {
1157            self.emit(RoderEvent::RunnerLifecycle(RunnerLifecycle {
1158                destination_id: "local".to_string(),
1159                provider_id: "local".to_string(),
1160                state: "local_fallback".to_string(),
1161                session_id: None,
1162                timestamp: OffsetDateTime::now_utc(),
1163            }))
1164            .await;
1165        }
1166    }
1167
1168    pub async fn set_file_backed_dynamic_context(&self, enabled: bool) -> RuntimeConfig {
1169        let mut cfg = self.config.write().await;
1170        cfg.file_backed_dynamic_context = enabled;
1171        cfg.clone()
1172    }
1173
1174    pub async fn set_command_shell(&self, shell: String) -> RuntimeConfig {
1175        let mut cfg = self.config.write().await;
1176        cfg.command_shell = shell;
1177        cfg.clone()
1178    }
1179
1180    pub async fn pending_plan_exit(&self) -> Option<PendingPlanExit> {
1181        let mut pending = self.pending_plan_exit.write().await;
1182        let current = pending.clone()?;
1183        if !current.is_expired(OffsetDateTime::now_utc()) {
1184            return Some(current);
1185        }
1186        *pending = None;
1187        drop(pending);
1188        self.emit_plan_exit_resolved(&current, false, self.status().await.policy_mode)
1189            .await;
1190        None
1191    }
1192
1193    pub async fn set_policy_mode(
1194        &self,
1195        mode: PolicyMode,
1196        reason: Option<String>,
1197    ) -> anyhow::Result<RuntimeConfig> {
1198        let mut cfg = self.config.write().await;
1199        let previous_mode = cfg.policy_mode;
1200        cfg.policy_mode = mode;
1201        let next = cfg.clone();
1202        drop(cfg);
1203        self.emit(RoderEvent::PolicyModeChanged(PolicyModeChanged {
1204            thread_id: "runtime".to_string(),
1205            turn_id: None,
1206            previous_mode,
1207            new_mode: mode,
1208            reason,
1209            timestamp: OffsetDateTime::now_utc(),
1210        }))
1211        .await;
1212        self.auto_resolve_pending_tool_approvals().await;
1213        Ok(next)
1214    }
1215
1216    /// Toggle agent-swarm mode for the runtime (roadmap 104). When enabled, the
1217    /// swarm reminder is injected into each turn's developer instructions so
1218    /// every app-server/SDK client gets it, not only the TUI.
1219    pub async fn set_agent_swarm_mode(
1220        &self,
1221        enabled: bool,
1222        trigger: roder_api::subagents::AgentSwarmModeTrigger,
1223    ) -> anyhow::Result<RuntimeConfig> {
1224        let mut cfg = self.config.write().await;
1225        cfg.agent_swarm_mode = enabled;
1226        let next = cfg.clone();
1227        drop(cfg);
1228        self.emit(RoderEvent::AgentSwarmModeChanged(
1229            roder_api::subagents::AgentSwarmModeChanged {
1230                thread_id: "runtime".to_string(),
1231                turn_id: None,
1232                enabled,
1233                trigger,
1234                timestamp: OffsetDateTime::now_utc(),
1235            },
1236        ))
1237        .await;
1238        Ok(next)
1239    }
1240
1241    /// Toggle agent-swarm mode for a single thread (roadmap 104). The override is
1242    /// stored per thread so toggling swarm mode on one thread does not leak the
1243    /// reminder into other threads sharing the runtime; absent threads fall back
1244    /// to the runtime-global default. The emitted `AgentSwarmModeChanged` event
1245    /// carries the real `thread_id`.
1246    pub async fn set_agent_swarm_mode_for_thread(
1247        &self,
1248        thread_id: &str,
1249        enabled: bool,
1250        trigger: roder_api::subagents::AgentSwarmModeTrigger,
1251    ) -> bool {
1252        {
1253            let mut modes = self.agent_swarm_modes.write().await;
1254            modes.insert(thread_id.to_string(), enabled);
1255        }
1256        self.emit(RoderEvent::AgentSwarmModeChanged(
1257            roder_api::subagents::AgentSwarmModeChanged {
1258                thread_id: thread_id.to_string(),
1259                turn_id: None,
1260                enabled,
1261                trigger,
1262                timestamp: OffsetDateTime::now_utc(),
1263            },
1264        ))
1265        .await;
1266        enabled
1267    }
1268
1269    /// Resolve whether agent-swarm mode is active for `thread_id`: the per-thread
1270    /// override when present, otherwise the runtime-global default. The turn loop
1271    /// uses this to decide whether to inject the swarm reminder.
1272    pub async fn effective_agent_swarm_mode_for_thread(&self, thread_id: &str) -> bool {
1273        if let Some(enabled) = self.agent_swarm_modes.read().await.get(thread_id).copied() {
1274            return enabled;
1275        }
1276        self.status().await.agent_swarm_mode
1277    }
1278
1279    /// Toggle ultra mode for the runtime. When enabled, proactive multi-agent
1280    /// policy is injected into each turn's developer instructions for any
1281    /// model, so every app-server/SDK client gets it (not only the TUI).
1282    pub async fn set_ultra_mode(
1283        &self,
1284        enabled: bool,
1285        trigger: roder_api::subagents::UltraModeTrigger,
1286    ) -> anyhow::Result<RuntimeConfig> {
1287        let mut cfg = self.config.write().await;
1288        cfg.ultra_mode = enabled;
1289        let next = cfg.clone();
1290        drop(cfg);
1291        self.emit(RoderEvent::UltraModeChanged(
1292            roder_api::subagents::UltraModeChanged {
1293                thread_id: "runtime".to_string(),
1294                turn_id: None,
1295                enabled,
1296                trigger,
1297                timestamp: OffsetDateTime::now_utc(),
1298            },
1299        ))
1300        .await;
1301        Ok(next)
1302    }
1303
1304    /// Toggle ultra mode for a single thread. The override is stored per thread
1305    /// so toggling on one thread does not leak proactive multi-agent policy into
1306    /// other threads sharing the runtime; absent threads fall back to the
1307    /// runtime-global default.
1308    pub async fn set_ultra_mode_for_thread(
1309        &self,
1310        thread_id: &str,
1311        enabled: bool,
1312        trigger: roder_api::subagents::UltraModeTrigger,
1313    ) -> bool {
1314        {
1315            let mut modes = self.ultra_modes.write().await;
1316            modes.insert(thread_id.to_string(), enabled);
1317        }
1318        self.emit(RoderEvent::UltraModeChanged(
1319            roder_api::subagents::UltraModeChanged {
1320                thread_id: thread_id.to_string(),
1321                turn_id: None,
1322                enabled,
1323                trigger,
1324                timestamp: OffsetDateTime::now_utc(),
1325            },
1326        ))
1327        .await;
1328        enabled
1329    }
1330
1331    /// Resolve whether ultra mode is active for `thread_id`: the per-thread
1332    /// override when present, otherwise the runtime-global default. The turn
1333    /// loop uses this (together with Sol/Terra Ultra effort) to decide whether
1334    /// to inject proactive multi-agent policy.
1335    pub async fn effective_ultra_mode_for_thread(&self, thread_id: &str) -> bool {
1336        if let Some(enabled) = self.ultra_modes.read().await.get(thread_id).copied() {
1337            return enabled;
1338        }
1339        self.status().await.ultra_mode
1340    }
1341
1342    pub async fn set_hosted_web_search(
1343        &self,
1344        mode: HostedWebSearchMode,
1345    ) -> anyhow::Result<RuntimeConfig> {
1346        let mut cfg = self.config.write().await;
1347        cfg.hosted_web_search = HostedWebSearchConfig { mode };
1348        Ok(cfg.clone())
1349    }
1350
1351    pub async fn record_pending_plan_exit(&self, pending: PendingPlanExit) {
1352        *self.pending_plan_exit.write().await = Some(pending.clone());
1353        self.emit(RoderEvent::PolicyExitPlanRequested(
1354            PolicyExitPlanRequested {
1355                thread_id: pending.thread_id,
1356                turn_id: pending.turn_id,
1357                request_id: pending.request_id,
1358                target_mode: pending.target_mode,
1359                plan_summary: pending.plan_summary,
1360                next_steps: pending.next_steps,
1361                timestamp: OffsetDateTime::now_utc(),
1362            },
1363        ))
1364        .await;
1365    }
1366
1367    pub async fn resolve_pending_plan_exit(
1368        &self,
1369        request_id: &str,
1370        approved: bool,
1371    ) -> anyhow::Result<Option<PendingPlanExit>> {
1372        let mut pending = self.pending_plan_exit.write().await;
1373        let Some(current) = pending.clone() else {
1374            return Ok(None);
1375        };
1376        if current.request_id != request_id {
1377            anyhow::bail!("pending plan exit request {request_id:?} was not found");
1378        }
1379        *pending = None;
1380        drop(pending);
1381
1382        let approved = approved && !current.is_expired(OffsetDateTime::now_utc());
1383        let resolved_mode = if approved {
1384            let mut cfg = self.config.write().await;
1385            let previous_mode = cfg.policy_mode;
1386            cfg.policy_mode = current.target_mode;
1387            drop(cfg);
1388            self.emit(RoderEvent::PolicyModeChanged(PolicyModeChanged {
1389                thread_id: current.thread_id.clone(),
1390                turn_id: Some(current.turn_id.clone()),
1391                previous_mode,
1392                new_mode: current.target_mode,
1393                reason: Some("approved plan exit".to_string()),
1394                timestamp: OffsetDateTime::now_utc(),
1395            }))
1396            .await;
1397            self.auto_resolve_pending_tool_approvals().await;
1398            current.target_mode
1399        } else {
1400            self.status().await.policy_mode
1401        };
1402        self.emit_plan_exit_resolved(&current, approved, resolved_mode)
1403            .await;
1404        Ok(Some(current))
1405    }
1406
1407    pub async fn resolve_tool_approval(
1408        &self,
1409        approval_id: &str,
1410        approved: bool,
1411    ) -> anyhow::Result<bool> {
1412        let pending = self.pending_tool_approvals.lock().await.remove(approval_id);
1413        let Some(pending) = pending else {
1414            return Ok(false);
1415        };
1416        self.emit(RoderEvent::ApprovalResolved(ApprovalResolved {
1417            thread_id: pending.thread_id,
1418            turn_id: pending.turn_id,
1419            approval_id: approval_id.to_string(),
1420            tool_id: pending.tool_id,
1421            tool_name: pending.tool_name,
1422            approved,
1423            timestamp: OffsetDateTime::now_utc(),
1424        }))
1425        .await;
1426        let _ = pending.tx.send(approved);
1427        Ok(true)
1428    }
1429
1430    /// Completes a pending host-executed tool call (`tools/resolve`). Returns false when the
1431    /// request id is unknown, already resolved, timed out, or cancelled by a turn interrupt.
1432    pub async fn resolve_external_tool_call(
1433        &self,
1434        request_id: &str,
1435        resolution: ExternalToolResolution,
1436    ) -> anyhow::Result<bool> {
1437        let mut calls = self.pending_external_tool_calls.lock().await;
1438        let Some(pending) = calls.get(request_id) else {
1439            return Ok(false);
1440        };
1441        self.emit_external_execution(RoderEvent::ExternalToolCallResolved(
1442            ExternalToolCallResolved {
1443                thread_id: pending.thread_id.clone(),
1444                turn_id: pending.turn_id.clone(),
1445                request_id: request_id.to_string(),
1446                tool_id: pending.tool_id.clone(),
1447                tool_name: pending.tool_name.clone(),
1448                outcome: ExternalToolCallOutcome::Resolved,
1449                is_error: resolution.is_error,
1450                timestamp: OffsetDateTime::now_utc(),
1451            },
1452        ))
1453        .await?;
1454        let Some(pending) = calls.remove(request_id) else {
1455            return Ok(false);
1456        };
1457        drop(calls);
1458        let _ = pending.tx.send(resolution);
1459        Ok(true)
1460    }
1461
1462    /// Drops every pending external tool call for the turn and reports them as cancelled so an
1463    /// interrupt does not leak requests waiting on `tools/resolve`.
1464    async fn cancel_pending_external_tool_calls_for_turn(&self, turn_id: &TurnId) {
1465        let cancelled = {
1466            let mut pending = self.pending_external_tool_calls.lock().await;
1467            let request_ids = pending
1468                .iter()
1469                .filter(|(_, call)| &call.turn_id == turn_id)
1470                .map(|(request_id, _)| request_id.clone())
1471                .collect::<Vec<_>>();
1472            request_ids
1473                .into_iter()
1474                .filter_map(|request_id| pending.remove(&request_id).map(|call| (request_id, call)))
1475                .collect::<Vec<_>>()
1476        };
1477        for (request_id, call) in cancelled {
1478            self.emit(RoderEvent::ExternalToolCallResolved(
1479                ExternalToolCallResolved {
1480                    thread_id: call.thread_id,
1481                    turn_id: call.turn_id,
1482                    request_id,
1483                    tool_id: call.tool_id,
1484                    tool_name: call.tool_name,
1485                    outcome: ExternalToolCallOutcome::Cancelled,
1486                    is_error: true,
1487                    timestamp: OffsetDateTime::now_utc(),
1488                },
1489            ))
1490            .await;
1491        }
1492    }
1493
1494    pub async fn resolve_user_input(
1495        &self,
1496        request_id: &str,
1497        answers: serde_json::Value,
1498    ) -> anyhow::Result<bool> {
1499        let pending = self.pending_user_inputs.lock().await.remove(request_id);
1500        let Some(pending) = pending else {
1501            return Ok(false);
1502        };
1503        self.emit(RoderEvent::UserInputResolved(UserInputResolved {
1504            thread_id: pending.thread_id,
1505            turn_id: pending.turn_id,
1506            request_id: request_id.to_string(),
1507            answers: answers.clone(),
1508            timestamp: OffsetDateTime::now_utc(),
1509        }))
1510        .await;
1511        let _ = pending.tx.send(answers);
1512        Ok(true)
1513    }
1514
1515    async fn emit_plan_exit_resolved(
1516        &self,
1517        current: &PendingPlanExit,
1518        approved: bool,
1519        resolved_mode: PolicyMode,
1520    ) {
1521        self.emit(RoderEvent::PolicyExitPlanResolved(PolicyExitPlanResolved {
1522            thread_id: current.thread_id.clone(),
1523            turn_id: current.turn_id.clone(),
1524            request_id: current.request_id.clone(),
1525            approved,
1526            target_mode: current.target_mode,
1527            resolved_mode,
1528            timestamp: OffsetDateTime::now_utc(),
1529        }))
1530        .await;
1531    }
1532
1533    pub async fn select_provider(
1534        &self,
1535        provider: String,
1536        model: Option<String>,
1537        reasoning: Option<String>,
1538    ) -> anyhow::Result<RuntimeConfig> {
1539        let next = self
1540            .preview_provider_selection(provider, model, reasoning)
1541            .await?;
1542        let mut cfg = self.config.write().await;
1543        *cfg = next;
1544        Ok(cfg.clone())
1545    }
1546
1547    pub async fn preview_provider_selection(
1548        &self,
1549        provider: String,
1550        model: Option<String>,
1551        reasoning: Option<String>,
1552    ) -> anyhow::Result<RuntimeConfig> {
1553        self.engine_for(&provider)?;
1554        let mut cfg = self.config.read().await.clone();
1555        cfg.default_provider = provider;
1556        if let Some(model) = model {
1557            cfg.default_model = model;
1558        }
1559        if let Some(reasoning) = reasoning {
1560            if reasoning == REASONING_NONE
1561                && !model_supports_reasoning(&cfg.default_model, &reasoning)
1562            {
1563                return Ok(cfg.clone());
1564            }
1565            validate_reasoning_effort(&cfg.default_model, &reasoning)?;
1566            cfg.reasoning = Some(reasoning);
1567        }
1568        Ok(cfg)
1569    }
1570
1571    pub async fn effective_reasoning(&self) -> String {
1572        let cfg = self.config.read().await;
1573        effective_reasoning_for_model(&cfg, &cfg.default_model)
1574    }
1575
1576    pub fn effective_reasoning_for_config(cfg: &RuntimeConfig) -> String {
1577        effective_reasoning_for_model(cfg, &cfg.default_model)
1578    }
1579
1580    pub async fn set_dynamic_workflow_effort(
1581        &self,
1582        effort_profile: DynamicWorkflowEffortProfile,
1583    ) -> RuntimeConfig {
1584        let mut cfg = self.config.write().await;
1585        cfg.dynamic_workflows.effort_profile = effort_profile;
1586        cfg.clone()
1587    }
1588
1589    pub async fn dynamic_workflow_trigger_decision(
1590        &self,
1591        message: &str,
1592    ) -> WorkflowTriggerDecision {
1593        let cfg = self.config.read().await;
1594        classify_workflow_trigger(message, &cfg.dynamic_workflows)
1595    }
1596
1597    pub async fn create_thread(&self, title: Option<String>) -> anyhow::Result<ThreadMetadata> {
1598        self.create_thread_with(CreateThreadRequest {
1599            title,
1600            workspace: self.workspace.display().to_string(),
1601            workspace_id: None,
1602            root_id: None,
1603            provider: None,
1604            model: None,
1605            selection_mode: None,
1606            tool_allowlist: Vec::new(),
1607            service_tier: None,
1608            developer_instructions: None,
1609            external_tools: Vec::new(),
1610            runner: None,
1611        })
1612        .await
1613    }
1614
1615    /**
1616     * Resolves an explicit thread-runner selection into a persisted binding.
1617     * Fails fast at thread creation when the provider is missing or rejects
1618     * the destination, instead of surfacing the error on the first tool call.
1619     */
1620    async fn resolve_thread_runner_binding(
1621        &self,
1622        thread_id: &str,
1623        selection: ThreadRunnerSelection,
1624    ) -> anyhow::Result<ThreadRunnerBinding> {
1625        let provider = self
1626            .registry
1627            .remote_runner_providers
1628            .iter()
1629            .find(|provider| provider.id() == selection.provider_id)
1630            .cloned()
1631            .ok_or_else(|| {
1632                anyhow::anyhow!(
1633                    "remote runner provider {:?} is not installed",
1634                    selection.provider_id
1635                )
1636            })?;
1637        let workspace = selection.workspace.trim();
1638        anyhow::ensure!(
1639            std::path::Path::new(workspace).is_absolute(),
1640            "runner workspace must be an absolute path on the runner: {workspace:?}"
1641        );
1642        let mut read_roots = Vec::with_capacity(selection.read_roots.len());
1643        for read_root in &selection.read_roots {
1644            let trimmed = read_root.trim();
1645            anyhow::ensure!(
1646                std::path::Path::new(trimmed).is_absolute(),
1647                "runner read root must be an absolute path on the runner: {trimmed:?}"
1648            );
1649            read_roots.push(PathBuf::from(trimmed));
1650        }
1651        let destination = RunnerDestination {
1652            id: format!("thread-{thread_id}"),
1653            provider_id: selection.provider_id,
1654            config: selection.config,
1655            default_manifest: roder_api::remote_runner::RunnerManifest::default(),
1656        };
1657        provider.validate_destination(&destination).await?;
1658        Ok(ThreadRunnerBinding {
1659            destination,
1660            workspace: PathBuf::from(workspace),
1661            read_roots,
1662        })
1663    }
1664
1665    /**
1666     * Build a per-thread binding from a runtime-level destination (selected via
1667     * the TUI runner picker or config `default_destination`) when the
1668     * destination's provider advertises a default workspace. Returns `None` when
1669     * there is no destination or the provider opts out (no default workspace),
1670     * preserving the legacy behavior where such threads keep local tools.
1671     */
1672    async fn synthesize_runtime_runner_binding(
1673        &self,
1674        thread_id: &str,
1675        destination: Option<RunnerDestination>,
1676    ) -> anyhow::Result<Option<ThreadRunnerBinding>> {
1677        let Some(destination) = destination else {
1678            return Ok(None);
1679        };
1680        let Some(provider) = self
1681            .registry
1682            .remote_runner_providers
1683            .iter()
1684            .find(|provider| provider.id() == destination.provider_id)
1685        else {
1686            return Ok(None);
1687        };
1688        // An explicit workspace in the destination config wins over the
1689        // provider default; absence of both means the provider opts out.
1690        let workspace = destination
1691            .config
1692            .get("working_dir")
1693            .and_then(serde_json::Value::as_str)
1694            .map(str::to_string)
1695            .or_else(|| provider.default_workspace());
1696        let Some(workspace) = workspace else {
1697            return Ok(None);
1698        };
1699        let selection = ThreadRunnerSelection {
1700            provider_id: destination.provider_id.clone(),
1701            config: destination.config.clone(),
1702            workspace,
1703            read_roots: Vec::new(),
1704        };
1705        Ok(Some(
1706            self.resolve_thread_runner_binding(thread_id, selection)
1707                .await?,
1708        ))
1709    }
1710
1711    /// Validates a runner selection without creating a thread; the placeholder destination id only appears in error messages.
1712    pub async fn validate_thread_runner_selection(
1713        &self,
1714        selection: ThreadRunnerSelection,
1715    ) -> anyhow::Result<()> {
1716        self.resolve_thread_runner_binding("validate", selection)
1717            .await
1718            .map(|_| ())
1719    }
1720
1721    pub async fn create_thread_with(
1722        &self,
1723        req: CreateThreadRequest,
1724    ) -> anyhow::Result<ThreadMetadata> {
1725        tool_advertisement::validate_external_tool_names(&req.external_tools)?;
1726        let cfg = self.config.read().await.clone();
1727        let now = OffsetDateTime::now_utc();
1728        let workspace = validate_thread_workspace(&req.workspace)?;
1729        let provider = req.provider.unwrap_or(cfg.default_provider);
1730        let model = req.model.unwrap_or(cfg.default_model);
1731        let selection_mode = req
1732            .selection_mode
1733            .unwrap_or_else(|| ModelSelectionMode::manual(provider.clone(), model.clone(), None));
1734        let thread_id = uuid::Uuid::new_v4().to_string();
1735        let runner_binding = match req.runner {
1736            Some(selection) => Some(
1737                self.resolve_thread_runner_binding(&thread_id, selection)
1738                    .await?,
1739            ),
1740            // No explicit per-thread selection: if a runtime-level destination
1741            // is active and its provider advertises a default workspace, bind
1742            // the new thread so its coding tools route into the runner. This is
1743            // what makes a TUI/config-selected runner (e.g. Blaxel) actually
1744            // execute tools in the sandbox instead of locally. Providers that
1745            // return no default workspace stay unbound (legacy local-tools).
1746            None => {
1747                self.synthesize_runtime_runner_binding(
1748                    &thread_id,
1749                    cfg.remote_runner_destination.clone(),
1750                )
1751                .await?
1752            }
1753        };
1754        let runner_destination = runner_binding
1755            .as_ref()
1756            .map(|binding| binding.destination.clone())
1757            .or_else(|| cfg.remote_runner_destination.clone());
1758        let metadata = ThreadMetadata {
1759            thread_id,
1760            title: req.title,
1761            workspace,
1762            workspace_id: req.workspace_id,
1763            root_id: req.root_id,
1764            provider: Some(provider),
1765            model: Some(model),
1766            selection_mode: Some(selection_mode),
1767            tool_allowlist: req.tool_allowlist,
1768            service_tier: req.service_tier,
1769            developer_instructions: req.developer_instructions,
1770            external_tools: req.external_tools,
1771            runner_destination,
1772            runner_state: None,
1773            runner_binding,
1774            created_at: now,
1775            updated_at: now,
1776            message_count: 0,
1777            usage: None,
1778            parent_thread_id: None,
1779            forked_from_turn_id: None,
1780            workspace_fork: None,
1781        };
1782
1783        let metadata = if let Some(store) = &self.thread_store {
1784            store.create_thread(metadata).await?
1785        } else {
1786            metadata
1787        };
1788        self.emit(RoderEvent::ThreadCreated(ThreadCreated {
1789            thread_id: metadata.thread_id.clone(),
1790            timestamp: OffsetDateTime::now_utc(),
1791        }))
1792        .await;
1793        Ok(metadata)
1794    }
1795
1796    pub async fn list_threads(&self) -> anyhow::Result<Vec<ThreadMetadata>> {
1797        if let Some(store) = &self.thread_store {
1798            return store.list_threads().await;
1799        }
1800        Ok(Vec::new())
1801    }
1802
1803    pub async fn list_threads_page(
1804        &self,
1805        options: roder_api::thread::ThreadListOptions,
1806    ) -> anyhow::Result<roder_api::thread::ThreadListPage> {
1807        if let Some(store) = &self.thread_store {
1808            return store.list_threads_page(options).await;
1809        }
1810        Ok(roder_api::thread::ThreadListPage::default())
1811    }
1812
1813    pub async fn load_thread_metadata(
1814        &self,
1815        thread_id: &str,
1816    ) -> anyhow::Result<Option<roder_api::thread::ThreadMetadata>> {
1817        if let Some(store) = &self.thread_store {
1818            return store.load_thread_metadata(&thread_id.to_string()).await;
1819        }
1820        Ok(None)
1821    }
1822
1823    pub async fn archive_thread(&self, thread_id: &str) -> anyhow::Result<bool> {
1824        let archived = if let Some(store) = &self.thread_store {
1825            store.archive_thread(&thread_id.to_string()).await?
1826        } else {
1827            false
1828        };
1829        if archived {
1830            let workspace = self.config.read().await.workspace.clone();
1831            crate::hooks::run_lifecycle(
1832                self,
1833                &thread_id.to_string(),
1834                &SESSION_HOOK_TURN_ID.to_string(),
1835                workspace.as_deref(),
1836                "SessionEnd",
1837                Some("clear"),
1838                serde_json::json!({"reason": "archive"}),
1839            )
1840            .await;
1841            self.thread_item_cache
1842                .lock()
1843                .await
1844                .remove_thread(&thread_id.to_string());
1845            self.evict_runner_session(&thread_id.to_string()).await;
1846        }
1847        Ok(archived)
1848    }
1849
1850    pub async fn start_team(&self, req: TeamStartRequest) -> anyhow::Result<TeamState> {
1851        let cfg = self.config.read().await.clone();
1852        let workspace = self.workspace.display().to_string();
1853        let lead_thread_id = match req.lead_thread_id {
1854            Some(thread_id) => thread_id,
1855            None => {
1856                self.create_thread_with(CreateThreadRequest {
1857                    title: Some("Team lead".to_string()),
1858                    workspace: workspace.clone(),
1859                    workspace_id: None,
1860                    root_id: None,
1861                    provider: None,
1862                    model: None,
1863                    selection_mode: None,
1864                    tool_allowlist: Vec::new(),
1865                    service_tier: None,
1866                    developer_instructions: None,
1867                    external_tools: Vec::new(),
1868                    runner: None,
1869                })
1870                .await?
1871                .thread_id
1872            }
1873        };
1874        let team_id = uuid::Uuid::new_v4().to_string();
1875        let active_lead_turn_id = self.active_turn_for_thread(&lead_thread_id).await;
1876        let lead_selection = match active_lead_turn_id.as_ref() {
1877            Some(turn_id) => self
1878                .active_turn_selections
1879                .read()
1880                .await
1881                .get(turn_id)
1882                .cloned(),
1883            None => self.selection_mode_for_thread(&lead_thread_id).await?,
1884        };
1885        let lead_concrete_selection = lead_selection
1886            .as_ref()
1887            .map(ModelSelectionMode::concrete_selection);
1888        let mut lead = crate::teams::lead_member(
1889            lead_thread_id.clone(),
1890            lead_concrete_selection
1891                .as_ref()
1892                .map(|selection| selection.provider.clone())
1893                .or_else(|| Some(cfg.default_provider.clone())),
1894            lead_concrete_selection
1895                .as_ref()
1896                .map(|selection| selection.model.clone())
1897                .or_else(|| Some(cfg.default_model.clone())),
1898            cfg.policy_mode,
1899        );
1900        if let Some(turn_id) = active_lead_turn_id {
1901            lead.current_turn_id = Some(turn_id);
1902            lead.status = TeamMemberStatus::Running;
1903        }
1904        let mut members = vec![lead];
1905
1906        for (index, member) in req.members.into_iter().enumerate() {
1907            let thread = self
1908                .create_thread_with(CreateThreadRequest {
1909                    title: Some(member.name.clone()),
1910                    workspace: workspace.clone(),
1911                    workspace_id: None,
1912                    root_id: None,
1913                    provider: member.model_provider.clone(),
1914                    model: member.model.clone(),
1915                    selection_mode: None,
1916                    tool_allowlist: Vec::new(),
1917                    service_tier: None,
1918                    developer_instructions: None,
1919                    external_tools: Vec::new(),
1920                    runner: None,
1921                })
1922                .await?;
1923            let member_id = format!("member-{}", index + 1);
1924            let descriptor = crate::teams::teammate_member(
1925                member_id.clone(),
1926                member.name,
1927                thread.thread_id.clone(),
1928                member.model_provider.or(thread.provider),
1929                member.model.or(thread.model),
1930                cfg.policy_mode,
1931            );
1932            members.push(descriptor);
1933        }
1934
1935        let now = OffsetDateTime::now_utc();
1936        let team = self
1937            .teams
1938            .insert(TeamState {
1939                id: team_id.clone(),
1940                lead_thread_id: lead_thread_id.clone(),
1941                display_mode: req.display_mode,
1942                members,
1943                mailbox: Vec::new(),
1944                tasks: Vec::new(),
1945                created_at: now,
1946                updated_at: now,
1947            })
1948            .await?;
1949        self.emit(RoderEvent::TeamStarted(TeamStarted {
1950            team_id: team_id.clone(),
1951            lead_thread_id,
1952            display_mode: team.display_mode,
1953            members: team.members.clone(),
1954            tasks: team.tasks.clone(),
1955            timestamp: OffsetDateTime::now_utc(),
1956        }))
1957        .await;
1958        for member in team
1959            .members
1960            .iter()
1961            .filter(|member| member.role != roder_api::teams::TeamMemberRole::Lead)
1962        {
1963            self.emit(RoderEvent::TeamMemberStarted(TeamMemberStarted {
1964                team_id: team_id.clone(),
1965                member: member.clone(),
1966                timestamp: OffsetDateTime::now_utc(),
1967            }))
1968            .await;
1969        }
1970        Ok(team)
1971    }
1972
1973    pub async fn list_teams(&self) -> Vec<TeamState> {
1974        let active_thread_ids = self
1975            .active_turns
1976            .read()
1977            .await
1978            .values()
1979            .map(|handle| handle.thread_id.clone())
1980            .collect::<std::collections::HashSet<_>>();
1981        self.teams
1982            .list()
1983            .await
1984            .into_iter()
1985            .map(|team| runtime_local_team_view(team, &active_thread_ids))
1986            .collect()
1987    }
1988
1989    pub async fn read_team(&self, team_id: &str) -> Option<TeamState> {
1990        let active_thread_ids = self
1991            .active_turns
1992            .read()
1993            .await
1994            .values()
1995            .map(|handle| handle.thread_id.clone())
1996            .collect::<std::collections::HashSet<_>>();
1997        self.teams
1998            .get(team_id)
1999            .await
2000            .map(|team| runtime_local_team_view(team, &active_thread_ids))
2001    }
2002
2003    pub async fn start_team_member(
2004        &self,
2005        team_id: &str,
2006        req: TeamMemberStartRequest,
2007    ) -> anyhow::Result<TeamState> {
2008        self.start_team_member_with_selection(team_id, req, None)
2009            .await
2010    }
2011
2012    pub async fn message_team_member(
2013        self: &Arc<Self>,
2014        team_id: &str,
2015        member_id: &str,
2016        message: String,
2017    ) -> anyhow::Result<TurnId> {
2018        let team = self
2019            .read_team(team_id)
2020            .await
2021            .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2022        self.followup_team_member(&team.lead_thread_id, team_id, member_id, message)
2023            .await
2024    }
2025
2026    /// Queue a mailbox message without starting an idle agent. If the target is
2027    /// already running, the message is delivered at the next inference boundary.
2028    pub(crate) async fn queue_team_member_message(
2029        self: &Arc<Self>,
2030        caller_thread_id: &ThreadId,
2031        team_id: &str,
2032        member_id: &str,
2033        message: String,
2034    ) -> anyhow::Result<Option<TurnId>> {
2035        let _delivery_guard = self.agent_team_spawn_lock.lock().await;
2036        let team = self
2037            .read_team(team_id)
2038            .await
2039            .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2040        let member = team
2041            .members
2042            .iter()
2043            .find(|member| member.id == member_id)
2044            .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2045            .clone();
2046        if member.status == TeamMemberStatus::Closed {
2047            anyhow::bail!("subagent {} is closed", member.name);
2048        }
2049        let from_member_id = team
2050            .members
2051            .iter()
2052            .find(|candidate| candidate.thread_id == *caller_thread_id)
2053            .map(|candidate| candidate.id.clone());
2054        self.teams
2055            .append_mailbox_message(
2056                team_id,
2057                from_member_id,
2058                member_id.to_string(),
2059                TeamMailboxMessageKind::Message,
2060                message,
2061            )
2062            .await?;
2063        let Some(turn_id) = self.active_turn_for_thread(&member.thread_id).await else {
2064            return Ok(None);
2065        };
2066        self.deliver_pending_team_mailbox(team_id, member_id, member.thread_id, turn_id.clone())
2067            .await?;
2068        Ok(Some(turn_id))
2069    }
2070
2071    /// Deliver queued messages and start an idle agent, or steer its active turn.
2072    pub(crate) async fn followup_team_member(
2073        self: &Arc<Self>,
2074        caller_thread_id: &ThreadId,
2075        team_id: &str,
2076        member_id: &str,
2077        message: String,
2078    ) -> anyhow::Result<TurnId> {
2079        let _spawn_guard = self.agent_team_spawn_lock.lock().await;
2080        self.followup_team_member_locked(caller_thread_id, team_id, member_id, message)
2081            .await
2082    }
2083
2084    async fn followup_team_member_locked(
2085        self: &Arc<Self>,
2086        caller_thread_id: &ThreadId,
2087        team_id: &str,
2088        member_id: &str,
2089        message: String,
2090    ) -> anyhow::Result<TurnId> {
2091        let team = self
2092            .read_team(team_id)
2093            .await
2094            .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2095        let member = team
2096            .members
2097            .iter()
2098            .find(|member| member.id == member_id)
2099            .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2100            .clone();
2101        if member.status == TeamMemberStatus::Closed {
2102            anyhow::bail!("subagent {} is closed", member.name);
2103        }
2104        let from_member_id = team
2105            .members
2106            .iter()
2107            .find(|candidate| candidate.thread_id == *caller_thread_id)
2108            .map(|candidate| candidate.id.clone());
2109        self.teams
2110            .append_mailbox_message(
2111                team_id,
2112                from_member_id,
2113                member_id.to_string(),
2114                TeamMailboxMessageKind::NewTask,
2115                message,
2116            )
2117            .await?;
2118        if let Some(turn_id) = self.active_turn_for_thread(&member.thread_id).await {
2119            self.deliver_pending_team_mailbox(
2120                team_id,
2121                member_id,
2122                member.thread_id,
2123                turn_id.clone(),
2124            )
2125            .await?;
2126            return Ok(turn_id);
2127        }
2128
2129        self.ensure_codex_v2_team_capacity(&team, &member.id, None)
2130            .await?;
2131        let metadata = self.load_thread_metadata(&member.thread_id).await?;
2132        let inherited_context = self
2133            .team_member_turn_contexts
2134            .lock()
2135            .await
2136            .get(&member.thread_id)
2137            .cloned();
2138        let workspace = inherited_context
2139            .as_ref()
2140            .map(|context| context.workspace.clone())
2141            .or_else(|| metadata.as_ref().map(|metadata| metadata.workspace.clone()))
2142            .unwrap_or_else(|| self.workspace.display().to_string());
2143        let member_thread_id = member.thread_id;
2144        let turn_id = self
2145            .start_turn(StartTurnRequest {
2146                thread_id: member_thread_id.clone(),
2147                message: String::new(),
2148                images: Vec::new(),
2149                provider_override: member.model_provider,
2150                model_override: member.model,
2151                reasoning_override: metadata
2152                    .as_ref()
2153                    .and_then(|metadata| metadata.selection_mode.as_ref())
2154                    .and_then(ModelSelectionMode::reasoning)
2155                    .map(str::to_string),
2156                workspace,
2157                instructions: inherited_context
2158                    .as_ref()
2159                    .map(|context| context.instructions.clone())
2160                    .unwrap_or_else(crate::default_instructions),
2161                developer_context: inherited_context
2162                    .as_ref()
2163                    .and_then(|context| context.developer_context.clone()),
2164                task_ledger_required: false,
2165                service_tier_override: None,
2166            })
2167            .await?;
2168        Ok(turn_id)
2169    }
2170
2171    pub async fn set_team_member_policy_mode(
2172        &self,
2173        team_id: &str,
2174        member_id: &str,
2175        policy_mode: PolicyMode,
2176    ) -> anyhow::Result<TeamState> {
2177        self.teams
2178            .set_member_policy_mode(team_id, member_id, policy_mode)
2179            .await
2180    }
2181
2182    pub async fn interrupt_team_member(
2183        &self,
2184        team_id: &str,
2185        member_id: &str,
2186    ) -> anyhow::Result<Option<TurnId>> {
2187        let _interrupt_guard = self.agent_team_spawn_lock.lock().await;
2188        let team = self
2189            .read_team(team_id)
2190            .await
2191            .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2192        let member = team
2193            .members
2194            .iter()
2195            .find(|member| member.id == member_id)
2196            .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2197            .clone();
2198        if member.status != TeamMemberStatus::Running {
2199            return Ok(None);
2200        }
2201        let Some(turn_id) = member.current_turn_id.clone() else {
2202            return Ok(None);
2203        };
2204        if self
2205            .active_turn_for_thread(&member.thread_id)
2206            .await
2207            .as_ref()
2208            != Some(&turn_id)
2209        {
2210            return Ok(None);
2211        }
2212        self.interrupt_turn(member.thread_id.clone(), turn_id.clone())
2213            .await?;
2214        // Make queued mailbox work immediately eligible for the replacement turn while the
2215        // lifecycle lock still prevents a follow-up from starting. Delivery acknowledgements
2216        // are turn-owned, so a late acknowledgement from the aborted turn cannot steal a
2217        // reservation acquired by its replacement.
2218        self.teams
2219            .release_mailbox_reservations_for_turn(&turn_id)
2220            .await;
2221        self.teams
2222            .update_member(team_id, member_id, |member| {
2223                member.status = TeamMemberStatus::Interrupted;
2224                member.current_turn_id = None;
2225                member.terminal_error = None;
2226            })
2227            .await?;
2228        self.emit(RoderEvent::TeamMemberCompleted(TeamMemberCompleted {
2229            team_id: team_id.to_string(),
2230            member_id: member_id.to_string(),
2231            member_thread_id: member.thread_id,
2232            turn_id: Some(turn_id.clone()),
2233            status: TeamMemberStatus::Interrupted,
2234            final_message: member.final_message,
2235            error: None,
2236            timestamp: OffsetDateTime::now_utc(),
2237        }))
2238        .await;
2239        Ok(Some(turn_id))
2240    }
2241
2242    pub async fn close_team_member(
2243        &self,
2244        team_id: &str,
2245        member_id: &str,
2246    ) -> anyhow::Result<roder_api::teams::TeamMemberDescriptor> {
2247        let team = self
2248            .read_team(team_id)
2249            .await
2250            .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
2251        let member = team
2252            .members
2253            .iter()
2254            .find(|member| member.id == member_id)
2255            .ok_or_else(|| anyhow::anyhow!("unknown team member {member_id:?}"))?
2256            .clone();
2257        if member.role == roder_api::teams::TeamMemberRole::Lead {
2258            anyhow::bail!("team lead cannot be closed as a subagent");
2259        }
2260        let interrupted_turn_id = if member.status == TeamMemberStatus::Running {
2261            if let Some(turn_id) = member.current_turn_id.clone() {
2262                self.interrupt_turn(member.thread_id.clone(), turn_id.clone())
2263                    .await?;
2264                Some(turn_id)
2265            } else {
2266                None
2267            }
2268        } else {
2269            member.current_turn_id.clone()
2270        };
2271        let updated = self
2272            .teams
2273            .update_member(team_id, member_id, |member| {
2274                member.status = TeamMemberStatus::Closed;
2275                member.current_turn_id = None;
2276            })
2277            .await?;
2278        let closed = updated
2279            .members
2280            .iter()
2281            .find(|member| member.id == member_id)
2282            .cloned()
2283            .ok_or_else(|| anyhow::anyhow!("closed team member disappeared"))?;
2284        self.team_member_turn_contexts
2285            .lock()
2286            .await
2287            .remove(&closed.thread_id);
2288        self.emit(RoderEvent::TeamMemberCompleted(TeamMemberCompleted {
2289            team_id: team_id.to_string(),
2290            member_id: closed.id.clone(),
2291            member_thread_id: closed.thread_id.clone(),
2292            turn_id: interrupted_turn_id,
2293            status: TeamMemberStatus::Closed,
2294            final_message: closed.final_message.clone(),
2295            error: closed.terminal_error.clone(),
2296            timestamp: OffsetDateTime::now_utc(),
2297        }))
2298        .await;
2299        Ok(closed)
2300    }
2301
2302    pub async fn cleanup_team(&self, team_id: &str, force: bool) -> anyhow::Result<bool> {
2303        let Some(team) = self.read_team(team_id).await else {
2304            return Ok(false);
2305        };
2306        if !force
2307            && team
2308                .members
2309                .iter()
2310                .any(|member| member.status == TeamMemberStatus::Running)
2311        {
2312            anyhow::bail!("team {team_id:?} has active teammates; use forced cleanup");
2313        }
2314        if force {
2315            for member in team.members.iter().filter(|member| {
2316                member.role != roder_api::teams::TeamMemberRole::Lead
2317                    && member.status == TeamMemberStatus::Running
2318            }) {
2319                if let Some(turn_id) = member
2320                    .current_turn_id
2321                    .clone()
2322                    .or(self.active_turn_for_thread(&member.thread_id).await)
2323                {
2324                    let _ = self.interrupt_turn(member.thread_id.clone(), turn_id).await;
2325                }
2326            }
2327        }
2328        let removed = self.teams.remove(team_id).await?.is_some();
2329        if removed {
2330            let member_thread_ids = team
2331                .members
2332                .iter()
2333                .map(|member| member.thread_id.clone())
2334                .collect::<std::collections::HashSet<_>>();
2335            self.team_member_turn_contexts
2336                .lock()
2337                .await
2338                .retain(|thread_id, _| !member_thread_ids.contains(thread_id));
2339            self.emit(RoderEvent::TeamCleanupCompleted(TeamCleanupCompleted {
2340                team_id: team_id.to_string(),
2341                forced: force,
2342                timestamp: OffsetDateTime::now_utc(),
2343            }))
2344            .await;
2345        }
2346        Ok(removed)
2347    }
2348
2349    pub async fn effective_policy_mode_for_thread(&self, thread_id: &str) -> PolicyMode {
2350        if let Some(mode) = self.teams.policy_mode_for_thread(thread_id).await {
2351            return mode;
2352        }
2353        self.status().await.policy_mode
2354    }
2355
2356    async fn complete_team_member_turn_with_result(
2357        &self,
2358        thread_id: &ThreadId,
2359        turn_id: &TurnId,
2360        status: TeamMemberStatus,
2361        final_message: Option<String>,
2362        terminal_error: Option<String>,
2363    ) -> anyhow::Result<()> {
2364        let _delivery_guard = self.agent_team_spawn_lock.lock().await;
2365        let Some((team_id, member)) = self
2366            .teams
2367            .complete_member_turn(
2368                thread_id,
2369                turn_id,
2370                status,
2371                final_message.clone(),
2372                terminal_error.clone(),
2373            )
2374            .await?
2375        else {
2376            return Ok(());
2377        };
2378        if let Some(parent_thread_id) = member.parent_thread_id.as_ref()
2379            && let Some(team) = self.read_team(&team_id).await
2380            && let Some(parent) = team
2381                .members
2382                .iter()
2383                .find(|candidate| candidate.thread_id == *parent_thread_id)
2384        {
2385            let identity = member
2386                .agent_path
2387                .as_deref()
2388                .or(member.task_name.as_deref())
2389                .unwrap_or(&member.name);
2390            let mut report = format!("Agent {identity} finished with status {status:?}.");
2391            if let Some(message) = final_message.as_deref()
2392                && !message.trim().is_empty()
2393            {
2394                report.push_str("\n\nFinal result:\n");
2395                report.push_str(message);
2396            }
2397            if let Some(error) = terminal_error.as_deref()
2398                && !error.trim().is_empty()
2399            {
2400                report.push_str("\n\nTerminal error:\n");
2401                report.push_str(error);
2402            }
2403            let mailbox_appended = self
2404                .teams
2405                .append_mailbox_message(
2406                    &team_id,
2407                    Some(member.id.clone()),
2408                    parent.id.clone(),
2409                    TeamMailboxMessageKind::FinalAnswer,
2410                    report,
2411                )
2412                .await
2413                .is_ok();
2414            if mailbox_appended
2415                && let Some(parent_turn_id) = self.active_turn_for_thread(parent_thread_id).await
2416            {
2417                // The parent may finish between the active-turn lookup and enqueue. Its durable
2418                // mailbox entry remains pending for the next turn, while terminal observers must
2419                // still receive TeamMemberCompleted for this child.
2420                let _ = self
2421                    .deliver_pending_team_mailbox(
2422                        &team_id,
2423                        &parent.id,
2424                        parent_thread_id.clone(),
2425                        parent_turn_id,
2426                    )
2427                    .await;
2428            }
2429        }
2430        self.emit(RoderEvent::TeamMemberCompleted(TeamMemberCompleted {
2431            team_id,
2432            member_id: member.id,
2433            member_thread_id: member.thread_id,
2434            turn_id: Some(turn_id.clone()),
2435            status,
2436            final_message,
2437            error: terminal_error,
2438            timestamp: OffsetDateTime::now_utc(),
2439        }))
2440        .await;
2441        Ok(())
2442    }
2443
2444    /// Stops accepting new turns, requests interruption for every active turn,
2445    /// and waits for their runtime tasks to reach terminal cleanup up to
2446    /// `timeout`. Repeated calls are safe and continue draining the same set.
2447    pub async fn drain_active_turns(&self, timeout: std::time::Duration) -> RuntimeDrainOutcome {
2448        let started_at = tokio::time::Instant::now();
2449        let persistence_failures_before =
2450            self.lifecycle_persistence_failures.load(Ordering::Acquire);
2451        let turn_admission = self.turn_admission.lock().await;
2452        self.accepting_turns.store(false, Ordering::Release);
2453        let active = self
2454            .active_turns
2455            .read()
2456            .await
2457            .iter()
2458            .map(|(turn_id, handle)| (turn_id.clone(), handle.thread_id.clone()))
2459            .collect::<Vec<_>>();
2460        let draining = self
2461            .turn_drains
2462            .read()
2463            .await
2464            .iter()
2465            .map(|(turn_id, drain)| (turn_id.clone(), drain.thread_id.clone()))
2466            .collect::<Vec<_>>();
2467        drop(turn_admission);
2468
2469        let mut interrupted_turns = std::collections::BTreeMap::new();
2470        for (turn_id, thread_id) in active.iter().chain(draining.iter()) {
2471            interrupted_turns.insert(turn_id.clone(), thread_id.clone());
2472        }
2473        let interrupted_turn_ids = interrupted_turns.keys().cloned().collect::<Vec<_>>();
2474        for (turn_id, thread_id) in active {
2475            let _ = self
2476                .interrupt_turn_with_reason(thread_id, turn_id, TurnLifecycleReason::Shutdown)
2477                .await;
2478        }
2479
2480        let wait_for_empty = async {
2481            loop {
2482                let notified = self.active_turns_changed.notified();
2483                if self.active_turns.read().await.is_empty()
2484                    && self.turn_drains.read().await.is_empty()
2485                {
2486                    return;
2487                }
2488                notified.await;
2489            }
2490        };
2491        let outcome = if tokio::time::timeout(timeout, wait_for_empty).await.is_ok() {
2492            if self.lifecycle_persistence_failures.load(Ordering::Acquire)
2493                > persistence_failures_before
2494            {
2495                RuntimeDrainOutcome::PersistenceFailed {
2496                    interrupted_turn_ids,
2497                    remaining_turn_ids: Vec::new(),
2498                }
2499            } else {
2500                RuntimeDrainOutcome::Clean {
2501                    interrupted_turn_ids,
2502                }
2503            }
2504        } else {
2505            let active_remaining = self
2506                .active_turns
2507                .read()
2508                .await
2509                .iter()
2510                .map(|(turn_id, handle)| (turn_id.clone(), handle.drain.clone()))
2511                .collect::<Vec<_>>();
2512            let draining_remaining = self
2513                .turn_drains
2514                .read()
2515                .await
2516                .iter()
2517                .map(|(turn_id, drain)| (turn_id.clone(), drain.clone()))
2518                .collect::<Vec<_>>();
2519            let remaining = active_remaining
2520                .into_iter()
2521                .chain(draining_remaining)
2522                .collect::<std::collections::BTreeMap<_, _>>();
2523            let remaining_turn_ids = remaining.keys().cloned().collect::<Vec<_>>();
2524            for turn_id in &remaining_turn_ids {
2525                let handle = remaining
2526                    .get(turn_id)
2527                    .expect("remaining turn ID must have a drain handle");
2528                self.record_turn_lifecycle(
2529                    handle.thread_id.clone(),
2530                    turn_id.clone(),
2531                    TurnLifecycleState::InterruptRequested,
2532                    TurnCleanupState::TimedOut,
2533                    Some(TurnLifecycleReason::Shutdown),
2534                )
2535                .await;
2536            }
2537            if self.lifecycle_persistence_failures.load(Ordering::Acquire)
2538                > persistence_failures_before
2539            {
2540                RuntimeDrainOutcome::PersistenceFailed {
2541                    interrupted_turn_ids,
2542                    remaining_turn_ids,
2543                }
2544            } else {
2545                RuntimeDrainOutcome::DeadlineExceeded {
2546                    interrupted_turn_ids,
2547                    remaining_turn_ids,
2548                }
2549            }
2550        };
2551        self.record_lifecycle_drain_outcome(started_at, &outcome);
2552        outcome
2553    }
2554
2555    /// Enables turn submission after a prior drain attempt. This is intended
2556    /// for embedders that keep a runtime alive after a bounded drain timeout.
2557    pub fn resume_accepting_turns(&self) {
2558        self.accepting_turns.store(true, Ordering::Release);
2559    }
2560
2561    pub async fn turn_lifecycle_snapshot(
2562        &self,
2563        thread_id: &ThreadId,
2564    ) -> anyhow::Result<TurnLifecycleSnapshot> {
2565        let Some(store) = &self.thread_store else {
2566            return Ok(TurnLifecycleSnapshot::default());
2567        };
2568        let extension_states = store.load_extension_states(thread_id).await?;
2569        Ok(turn_lifecycle_snapshot(&extension_states))
2570    }
2571
2572    pub async fn load_thread(
2573        &self,
2574        thread_id: &ThreadId,
2575    ) -> anyhow::Result<Option<ThreadSnapshot>> {
2576        let loaded = if let Some(store) = &self.thread_store {
2577            store.load_thread(thread_id).await?
2578        } else {
2579            None
2580        };
2581        if let Some(snapshot) = loaded.as_ref() {
2582            let live_turn_ids = self
2583                .active_turns
2584                .read()
2585                .await
2586                .keys()
2587                .cloned()
2588                .chain(self.turn_drains.read().await.keys().cloned())
2589                .collect::<std::collections::HashSet<_>>();
2590            let lifecycle = turn_lifecycle_snapshot(&snapshot.extension_states);
2591            for record in lifecycle.records.into_iter().filter(|record| {
2592                record.state.requires_recovery() && !live_turn_ids.contains(&record.turn_id)
2593            }) {
2594                self.lifecycle_restart_reconciliations
2595                    .fetch_add(1, Ordering::AcqRel);
2596                self.record_turn_lifecycle(
2597                    record.thread_id,
2598                    record.turn_id,
2599                    TurnLifecycleState::RecoveryNeeded,
2600                    TurnCleanupState::Unknown,
2601                    Some(TurnLifecycleReason::RuntimeRestart),
2602                )
2603                .await;
2604            }
2605            self.emit(RoderEvent::ThreadLoaded(ThreadLoaded {
2606                thread_id: thread_id.clone(),
2607                timestamp: OffsetDateTime::now_utc(),
2608            }))
2609            .await;
2610        }
2611        Ok(loaded)
2612    }
2613
2614    pub async fn workspace_for_thread(&self, thread_id: &ThreadId) -> anyhow::Result<String> {
2615        if let Some(store) = &self.thread_store {
2616            let snapshot = store
2617                .load_thread(thread_id)
2618                .await?
2619                .ok_or_else(|| anyhow::anyhow!("thread not found: {thread_id}"));
2620            match snapshot {
2621                Ok(snapshot) => {
2622                    if let Some(metadata) = snapshot.metadata {
2623                        // Fork-backed threads fail closed before any write
2624                        // when their workspace was removed out-of-band.
2625                        if let Some(fork) = &metadata.workspace_fork
2626                            && fork.status == roder_api::forks::ForkStatus::Active
2627                            && !std::path::Path::new(&metadata.workspace).is_dir()
2628                        {
2629                            anyhow::bail!(
2630                                "workspace fork {} is missing its workspace at {}; restore it or \
2631                                 remove the fork before running turns in this thread",
2632                                fork.id,
2633                                metadata.workspace
2634                            );
2635                        }
2636                        return Ok(metadata.workspace);
2637                    }
2638                    eprintln!(
2639                        "thread metadata missing while resolving workspace for {thread_id}; falling back to runtime workspace"
2640                    );
2641                }
2642                Err(err) => {
2643                    eprintln!(
2644                        "thread missing while resolving workspace for {thread_id}: {err}; falling back to runtime workspace"
2645                    );
2646                }
2647            }
2648        }
2649        Ok(self.workspace.display().to_string())
2650    }
2651
2652    pub(crate) async fn selection_mode_for_thread(
2653        &self,
2654        thread_id: &ThreadId,
2655    ) -> anyhow::Result<Option<ModelSelectionMode>> {
2656        let Some(store) = &self.thread_store else {
2657            return Ok(None);
2658        };
2659        Ok(store
2660            .load_thread_metadata(thread_id)
2661            .await?
2662            .and_then(|metadata| {
2663                metadata
2664                    .selection_mode
2665                    .or_else(|| match (metadata.provider, metadata.model) {
2666                        (Some(provider), Some(model)) => {
2667                            Some(ModelSelectionMode::manual(provider, model, None))
2668                        }
2669                        _ => None,
2670                    })
2671            }))
2672    }
2673
2674    /// Concrete provider/model for configured-role subagents (`task`,
2675    /// `agent_swarm`). Prefers the live parent turn selection so children stay
2676    /// on SuperGrok/Grok (or whatever the lead is using) instead of process
2677    /// startup defaults such as openai/gpt-5.5.
2678    pub(crate) async fn parent_model_selection_for_subagents(
2679        &self,
2680        thread_id: &ThreadId,
2681        turn_id: &TurnId,
2682    ) -> Option<roder_api::inference::ModelSelection> {
2683        if let Some(selection) = self
2684            .active_turn_selections
2685            .read()
2686            .await
2687            .get(turn_id)
2688            .cloned()
2689        {
2690            return Some(selection.concrete_selection());
2691        }
2692        if let Ok(Some(selection)) = self.selection_mode_for_thread(thread_id).await {
2693            return Some(selection.concrete_selection());
2694        }
2695        let cfg = self.status().await;
2696        if cfg.default_provider.trim().is_empty() || cfg.default_model.trim().is_empty() {
2697            return None;
2698        }
2699        Some(roder_api::inference::ModelSelection {
2700            provider: cfg.default_provider,
2701            model: cfg.default_model,
2702        })
2703    }
2704
2705    /// Per-thread tool allowlist, developer instructions, and external tools persisted at thread creation.
2706    pub(crate) async fn thread_turn_overrides(
2707        &self,
2708        thread_id: &ThreadId,
2709    ) -> anyhow::Result<ThreadTurnOverrides> {
2710        let Some(store) = &self.thread_store else {
2711            return Ok(ThreadTurnOverrides::default());
2712        };
2713        Ok(store
2714            .load_thread_metadata(thread_id)
2715            .await?
2716            .map(|metadata| ThreadTurnOverrides {
2717                service_tier: metadata.service_tier,
2718                tool_allowlist: metadata.tool_allowlist,
2719                developer_instructions: metadata.developer_instructions,
2720                external_tools: metadata.external_tools,
2721            })
2722            .unwrap_or_default())
2723    }
2724
2725    pub async fn set_thread_selection_mode(
2726        &self,
2727        thread_id: &ThreadId,
2728        selection_mode: ModelSelectionMode,
2729    ) -> anyhow::Result<()> {
2730        let Some(store) = &self.thread_store else {
2731            return Ok(());
2732        };
2733        let Some(mut metadata) = store.load_thread_metadata(thread_id).await? else {
2734            anyhow::bail!("thread not found: {thread_id}");
2735        };
2736        let concrete = selection_mode.concrete_selection();
2737        metadata.provider = Some(concrete.provider);
2738        metadata.model = Some(concrete.model);
2739        metadata.selection_mode = Some(selection_mode);
2740        metadata.updated_at = OffsetDateTime::now_utc();
2741        store.update_thread_metadata(metadata).await?;
2742        Ok(())
2743    }
2744
2745    async fn runner_session_for_thread(
2746        &self,
2747        thread_id: &ThreadId,
2748    ) -> anyhow::Result<Option<(RunnerDestination, Arc<dyn RemoteRunnerSession>)>> {
2749        let metadata = if let Some(store) = &self.thread_store {
2750            store.load_thread_metadata(thread_id).await?
2751        } else {
2752            None
2753        };
2754        // An explicit per-thread binding wins over the runtime-level destination.
2755        let destination = metadata
2756            .as_ref()
2757            .and_then(|metadata| metadata.runner_binding.as_ref())
2758            .map(|binding| binding.destination.clone())
2759            .or(self.config.read().await.remote_runner_destination.clone());
2760        let Some(destination) = destination else {
2761            self.evict_runner_session(thread_id).await;
2762            return Ok(None);
2763        };
2764        let provider = self
2765            .registry
2766            .remote_runner_providers
2767            .iter()
2768            .find(|provider| provider.id() == destination.provider_id)
2769            .cloned()
2770            .ok_or_else(|| {
2771                anyhow::anyhow!(
2772                    "remote runner provider {:?} is not installed",
2773                    destination.provider_id
2774                )
2775            })?;
2776        let persisted_state = metadata.and_then(|metadata| metadata.runner_state);
2777        let slot = {
2778            let mut sessions = self.runner_sessions.lock().await;
2779            match sessions.get(thread_id) {
2780                Some(slot) if slot.matches(&destination) => slot.clone(),
2781                _ => {
2782                    let slot = Arc::new(RunnerSessionSlot::empty(&destination));
2783                    sessions.insert(thread_id.clone(), slot.clone());
2784                    slot
2785                }
2786            }
2787        };
2788
2789        loop {
2790            let initialized = slot.initialized.notified();
2791            let mut state = slot.state.lock().await;
2792            match &*state {
2793                RunnerSessionSlotState::Ready(cached) => {
2794                    return Ok(Some((cached.destination.clone(), cached.session.clone())));
2795                }
2796                RunnerSessionSlotState::Failed(error) => {
2797                    return Err(anyhow::anyhow!(error.to_string()));
2798                }
2799                RunnerSessionSlotState::Initializing => {
2800                    drop(state);
2801                    initialized.await;
2802                }
2803                RunnerSessionSlotState::Empty => {
2804                    *state = RunnerSessionSlotState::Initializing;
2805                    drop(state);
2806                    self.spawn_runner_session_initialization(
2807                        thread_id.clone(),
2808                        destination.clone(),
2809                        provider.clone(),
2810                        persisted_state.clone(),
2811                        slot.clone(),
2812                    );
2813                }
2814            }
2815        }
2816    }
2817
2818    fn spawn_runner_session_initialization(
2819        &self,
2820        thread_id: ThreadId,
2821        destination: RunnerDestination,
2822        provider: Arc<dyn RemoteRunnerProvider>,
2823        persisted_state: Option<RunnerSessionState>,
2824        slot: Arc<RunnerSessionSlot>,
2825    ) {
2826        let thread_store = self.thread_store.clone();
2827        let runner_sessions = self.runner_sessions.clone();
2828        // The task intentionally outlives the turn that first requested the
2829        // workspace. Interrupting that turn must not strand the slot in
2830        // `Initializing` and deadlock every later tool or lifecycle call.
2831        drop(tokio::spawn(async move {
2832            let initialized = async {
2833                let session = if let Some(state) = persisted_state
2834                    && state.provider_id == destination.provider_id
2835                    && state.destination_id == destination.id
2836                {
2837                    match provider.resume_session(state).await {
2838                        Ok(session) => session,
2839                        Err(_) => provider.create_session(destination.clone()).await?,
2840                    }
2841                } else {
2842                    provider.create_session(destination.clone()).await?
2843                };
2844                let resolved = (destination.clone(), session.clone());
2845                // Persist before releasing the singleflight or dispatching
2846                // the first tool. A later process can rejoin even if the turn
2847                // that requested initialization has already been interrupted.
2848                Self::persist_runner_state_in_store(
2849                    thread_store.as_ref(),
2850                    &thread_id,
2851                    Some(&resolved),
2852                )
2853                .await?;
2854                Ok::<_, anyhow::Error>(CachedRunnerSession {
2855                    destination,
2856                    session,
2857                })
2858            }
2859            .await;
2860
2861            match initialized {
2862                Ok(cached) => {
2863                    *slot.state.lock().await = RunnerSessionSlotState::Ready(cached);
2864                    slot.initialized.notify_waiters();
2865                }
2866                Err(error) => {
2867                    let message: Arc<str> = Arc::from(format!("{error:#}"));
2868                    *slot.state.lock().await = RunnerSessionSlotState::Failed(message);
2869                    slot.initialized.notify_waiters();
2870                    let mut sessions = runner_sessions.lock().await;
2871                    let is_current = sessions
2872                        .get(&thread_id)
2873                        .is_some_and(|current| Arc::ptr_eq(current, &slot));
2874                    if is_current {
2875                        sessions.remove(&thread_id);
2876                    }
2877                }
2878            }
2879        }));
2880    }
2881
2882    async fn evict_runner_session(&self, thread_id: &ThreadId) {
2883        self.runner_sessions.lock().await.remove(thread_id);
2884    }
2885
2886    async fn cache_runner_session(
2887        &self,
2888        thread_id: &ThreadId,
2889        destination: RunnerDestination,
2890        session: Arc<dyn RemoteRunnerSession>,
2891    ) {
2892        let cached = CachedRunnerSession {
2893            destination,
2894            session,
2895        };
2896        self.runner_sessions.lock().await.insert(
2897            thread_id.clone(),
2898            Arc::new(RunnerSessionSlot::ready(cached)),
2899        );
2900    }
2901
2902    /// Read a thread's explicit runner binding without creating or resuming a
2903    /// session. Tool routing uses this before local previews so runner-backed
2904    /// and fail-closed hosted calls never inspect the host workspace.
2905    pub(crate) async fn runner_binding_for_thread(
2906        &self,
2907        thread_id: &ThreadId,
2908    ) -> anyhow::Result<Option<ThreadRunnerBinding>> {
2909        let Some(store) = &self.thread_store else {
2910            return Ok(None);
2911        };
2912        Ok(store
2913            .load_thread_metadata(thread_id)
2914            .await?
2915            .and_then(|metadata| metadata.runner_binding))
2916    }
2917
2918    /// Resolve a previously-read binding into a live remote workspace. The
2919    /// session remains lazy: callers invoke this only after policy approval.
2920    pub(crate) async fn remote_workspace_for_binding(
2921        &self,
2922        thread_id: &ThreadId,
2923        binding: ThreadRunnerBinding,
2924    ) -> anyhow::Result<Arc<RemoteWorkspace>> {
2925        let session = self
2926            .runner_session_for_thread(thread_id)
2927            .await?
2928            .map(|(_, session)| session)
2929            .ok_or_else(|| {
2930                anyhow::anyhow!("runner-bound thread {thread_id} has no runner session")
2931            })?;
2932        Ok(Arc::new(RemoteWorkspace {
2933            session,
2934            root: binding.workspace,
2935            read_roots: binding.read_roots,
2936        }))
2937    }
2938
2939    pub(crate) async fn runner_tool_execution_lock(
2940        &self,
2941        session: &dyn RemoteRunnerSession,
2942    ) -> Arc<RunnerToolExecutionMutex> {
2943        let state = session.state();
2944        let key = (state.provider_id, state.session_id);
2945        let mut locks = self.runner_tool_execution_locks.lock().await;
2946        if let Some(lock) = locks.get(&key).and_then(Weak::upgrade) {
2947            return lock;
2948        }
2949
2950        locks.retain(|_, lock| lock.strong_count() > 0);
2951        let lock = Arc::new(Mutex::new(()));
2952        locks.insert(key, Arc::downgrade(&lock));
2953        lock
2954    }
2955
2956    async fn persist_runner_state(
2957        &self,
2958        thread_id: &ThreadId,
2959        runner: Option<&(RunnerDestination, Arc<dyn RemoteRunnerSession>)>,
2960    ) -> anyhow::Result<()> {
2961        Self::persist_runner_state_in_store(self.thread_store.as_ref(), thread_id, runner).await
2962    }
2963
2964    async fn persist_runner_state_in_store(
2965        store: Option<&Arc<dyn ThreadStore>>,
2966        thread_id: &ThreadId,
2967        runner: Option<&(RunnerDestination, Arc<dyn RemoteRunnerSession>)>,
2968    ) -> anyhow::Result<()> {
2969        let Some((destination, session)) = runner else {
2970            return Ok(());
2971        };
2972        let Some(store) = store else {
2973            return Ok(());
2974        };
2975        let Some(snapshot) = store.load_thread(thread_id).await? else {
2976            return Ok(());
2977        };
2978        let Some(mut metadata) = snapshot.metadata else {
2979            return Ok(());
2980        };
2981        metadata.runner_destination = Some(destination.clone());
2982        metadata.runner_state = Some(session.state());
2983        metadata.updated_at = OffsetDateTime::now_utc();
2984        store.update_thread_metadata(metadata).await?;
2985        Ok(())
2986    }
2987
2988    fn remote_runner_provider_by_id(
2989        &self,
2990        provider_id: &str,
2991    ) -> Option<Arc<dyn RemoteRunnerProvider>> {
2992        self.registry
2993            .remote_runner_providers
2994            .iter()
2995            .find(|provider| provider.id() == provider_id)
2996            .cloned()
2997    }
2998
2999    /// Resolve the live runner session for a thread along with its provider,
3000    /// failing clearly when the thread is not runner-bound.
3001    async fn thread_runner_session(
3002        &self,
3003        thread_id: &ThreadId,
3004    ) -> anyhow::Result<(
3005        RunnerDestination,
3006        Arc<dyn RemoteRunnerProvider>,
3007        Arc<dyn RemoteRunnerSession>,
3008    )> {
3009        let Some((destination, session)) = self.runner_session_for_thread(thread_id).await? else {
3010            anyhow::bail!("thread {thread_id} is not bound to a remote runner");
3011        };
3012        let provider = self
3013            .remote_runner_provider_by_id(&destination.provider_id)
3014            .ok_or_else(|| {
3015                anyhow::anyhow!(
3016                    "remote runner provider {:?} is not installed",
3017                    destination.provider_id
3018                )
3019            })?;
3020        Ok((destination, provider, session))
3021    }
3022
3023    /// Pause a runner-bound thread's session toward standby and persist the
3024    /// post-pause state. Errors if the provider is not pausable.
3025    pub async fn pause_thread_runner(
3026        &self,
3027        thread_id: &ThreadId,
3028    ) -> anyhow::Result<RunnerSessionState> {
3029        let (destination, provider, session) = self.thread_runner_session(thread_id).await?;
3030        anyhow::ensure!(
3031            provider.capabilities().pausable,
3032            "remote runner provider {:?} does not support pausing",
3033            destination.provider_id
3034        );
3035        let state = session.pause().await?;
3036        self.persist_runner_state(thread_id, Some(&(destination, session)))
3037            .await?;
3038        Ok(state)
3039    }
3040
3041    /// Resume (wake) a runner-bound thread's paused session and persist state.
3042    pub async fn resume_thread_runner(
3043        &self,
3044        thread_id: &ThreadId,
3045    ) -> anyhow::Result<RunnerSessionState> {
3046        let (destination, _provider, session) = self.thread_runner_session(thread_id).await?;
3047        let state = session.resume().await?;
3048        self.persist_runner_state(thread_id, Some(&(destination, session)))
3049            .await?;
3050        Ok(state)
3051    }
3052
3053    /// Detach a runner-bound thread's session: persist the durable, rejoinable
3054    /// state and leave the remote sandbox alive. Errors if not detachable.
3055    pub async fn detach_thread_runner(
3056        &self,
3057        thread_id: &ThreadId,
3058    ) -> anyhow::Result<RunnerSessionState> {
3059        let (destination, provider, session) = self.thread_runner_session(thread_id).await?;
3060        anyhow::ensure!(
3061            provider.capabilities().detachable,
3062            "remote runner provider {:?} does not support detaching",
3063            destination.provider_id
3064        );
3065        let state = session.detach().await?;
3066        // Persist the detached state explicitly so a later turn or process
3067        // rejoins the same sandbox instead of provisioning a new one.
3068        if let Some(store) = &self.thread_store
3069            && let Some(snapshot) = store.load_thread(thread_id).await?
3070            && let Some(mut metadata) = snapshot.metadata
3071        {
3072            metadata.runner_destination = Some(destination);
3073            metadata.runner_state = Some(state.clone());
3074            metadata.updated_at = OffsetDateTime::now_utc();
3075            store.update_thread_metadata(metadata).await?;
3076        }
3077        self.evict_runner_session(thread_id).await;
3078        Ok(state)
3079    }
3080
3081    /// Rejoin a previously created sandbox from a thread's persisted runner
3082    /// state without provisioning a new one. An optional `sandbox` overrides the
3083    /// persisted sandbox name (recovery by name). Persists refreshed state.
3084    pub async fn rejoin_thread_runner(
3085        &self,
3086        thread_id: &ThreadId,
3087        sandbox: Option<String>,
3088    ) -> anyhow::Result<RunnerSessionState> {
3089        let store = self
3090            .thread_store
3091            .as_ref()
3092            .ok_or_else(|| anyhow::anyhow!("thread store is required to rejoin a runner"))?;
3093        let metadata = store
3094            .load_thread_metadata(thread_id)
3095            .await?
3096            .ok_or_else(|| anyhow::anyhow!("thread {thread_id} has no metadata"))?;
3097        let destination = metadata
3098            .runner_binding
3099            .as_ref()
3100            .map(|binding| binding.destination.clone())
3101            .or_else(|| metadata.runner_destination.clone())
3102            .ok_or_else(|| anyhow::anyhow!("thread {thread_id} is not bound to a remote runner"))?;
3103        let mut state = metadata
3104            .runner_state
3105            .clone()
3106            .ok_or_else(|| anyhow::anyhow!("thread {thread_id} has no persisted runner state"))?;
3107        if let Some(sandbox) = sandbox
3108            && let Some(object) = state.metadata.as_object_mut()
3109        {
3110            object.insert("sandbox_name".to_string(), serde_json::Value::from(sandbox));
3111        }
3112        let provider = self
3113            .remote_runner_provider_by_id(&destination.provider_id)
3114            .ok_or_else(|| {
3115                anyhow::anyhow!(
3116                    "remote runner provider {:?} is not installed",
3117                    destination.provider_id
3118                )
3119            })?;
3120        let session = provider.rejoin_session(state).await?;
3121        self.persist_runner_state(thread_id, Some(&(destination.clone(), session.clone())))
3122            .await?;
3123        self.cache_runner_session(thread_id, destination, session.clone())
3124            .await;
3125        Ok(session.state())
3126    }
3127
3128    pub(crate) async fn record_thread_usage_metadata(
3129        &self,
3130        thread_id: &ThreadId,
3131        usage: &TokenUsage,
3132    ) -> anyhow::Result<()> {
3133        if usage.is_empty() {
3134            return Ok(());
3135        }
3136        let _guard = self.thread_usage_metadata_lock.lock().await;
3137        let Some(store) = &self.thread_store else {
3138            return Ok(());
3139        };
3140        let Some(snapshot) = store.load_thread(thread_id).await? else {
3141            return Ok(());
3142        };
3143        let Some(mut metadata) = snapshot.metadata else {
3144            return Ok(());
3145        };
3146        metadata
3147            .usage
3148            .get_or_insert_with(ThreadUsageMetadata::default)
3149            .add_token_usage(usage);
3150        metadata.updated_at = OffsetDateTime::now_utc();
3151        store.update_thread_metadata(metadata).await?;
3152        Ok(())
3153    }
3154
3155    pub(crate) async fn has_active_turn_for_thread(&self, thread_id: &ThreadId) -> bool {
3156        self.active_turns
3157            .read()
3158            .await
3159            .values()
3160            .any(|handle| &handle.thread_id == thread_id)
3161    }
3162
3163    async fn ensure_codex_v2_team_capacity(
3164        &self,
3165        team: &TeamState,
3166        resuming_member_id: &str,
3167        candidate_model: Option<&str>,
3168    ) -> anyhow::Result<()> {
3169        if !is_codex_v2_team(team)
3170            && !candidate_model
3171                .is_some_and(|model| model_supports_reasoning_effort(model, REASONING_ULTRA))
3172        {
3173            return Ok(());
3174        }
3175        let active_thread_ids = self
3176            .active_turns
3177            .read()
3178            .await
3179            .values()
3180            .map(|handle| handle.thread_id.clone())
3181            .collect::<std::collections::HashSet<_>>();
3182        let resident_threads = 1 + team
3183            .members
3184            .iter()
3185            .filter(|member| {
3186                member.role != roder_api::teams::TeamMemberRole::Lead
3187                    && member.id != resuming_member_id
3188                    && active_thread_ids.contains(&member.thread_id)
3189            })
3190            .count();
3191        anyhow::ensure!(
3192            resident_threads < codex_v2::CODEX_V2_MAX_RESIDENT_TEAM_THREADS,
3193            "agent thread limit reached for this Codex V2 team: maximum {} resident threads (the lead plus 3 running subagents)",
3194            codex_v2::CODEX_V2_MAX_RESIDENT_TEAM_THREADS
3195        );
3196        Ok(())
3197    }
3198
3199    /// Number of currently running turns across all threads. Hosted runtime
3200    /// pools use this to avoid evicting tenants with active work.
3201    pub async fn active_turn_count(&self) -> usize {
3202        self.active_turns.read().await.len()
3203    }
3204
3205    pub async fn active_turn_for_thread(&self, thread_id: &ThreadId) -> Option<TurnId> {
3206        self.active_turns
3207            .read()
3208            .await
3209            .iter()
3210            .find_map(|(turn_id, handle)| (&handle.thread_id == thread_id).then(|| turn_id.clone()))
3211    }
3212
3213    pub async fn thread_activity(&self, thread_id: &ThreadId) -> ThreadActivity {
3214        let active_turn_id = self.active_turn_for_thread(thread_id).await;
3215        let draining_turn_id = if active_turn_id.is_none() {
3216            self.turn_drains
3217                .read()
3218                .await
3219                .iter()
3220                .find_map(|(turn_id, drain)| {
3221                    (&drain.thread_id == thread_id).then(|| turn_id.clone())
3222                })
3223        } else {
3224            None
3225        };
3226        let Some(active_turn_id) = active_turn_id.or(draining_turn_id) else {
3227            return ThreadActivity::default();
3228        };
3229
3230        let mut active_flags = Vec::new();
3231        {
3232            let pending_approvals = self.pending_tool_approvals.lock().await;
3233            if pending_approvals
3234                .values()
3235                .any(|pending| &pending.thread_id == thread_id && pending.turn_id == active_turn_id)
3236            {
3237                active_flags.push("approvalRequired".to_string());
3238            }
3239        }
3240        {
3241            let pending_inputs = self.pending_user_inputs.lock().await;
3242            if pending_inputs
3243                .values()
3244                .any(|pending| &pending.thread_id == thread_id && pending.turn_id == active_turn_id)
3245            {
3246                active_flags.push("userInputRequired".to_string());
3247            }
3248        }
3249        {
3250            let pending_external = self.pending_external_tool_calls.lock().await;
3251            if pending_external
3252                .values()
3253                .any(|pending| &pending.thread_id == thread_id && pending.turn_id == active_turn_id)
3254            {
3255                active_flags.push("externalToolPending".to_string());
3256            }
3257        }
3258        let interrupting = self
3259            .active_turns
3260            .read()
3261            .await
3262            .get(&active_turn_id)
3263            .is_some_and(|handle| handle.drain.interrupt_requested.load(Ordering::Acquire))
3264            || self
3265                .turn_drains
3266                .read()
3267                .await
3268                .get(&active_turn_id)
3269                .is_some_and(|drain| drain.interrupt_requested.load(Ordering::Acquire));
3270        if interrupting {
3271            active_flags.push("interrupting".to_string());
3272        }
3273        if self.pending_plan_exit().await.is_some_and(|pending| {
3274            &pending.thread_id == thread_id && pending.turn_id == active_turn_id
3275        }) {
3276            active_flags.push("planExitRequired".to_string());
3277        }
3278
3279        ThreadActivity {
3280            active_turn_id: Some(active_turn_id),
3281            active_flags,
3282        }
3283    }
3284
3285    pub async fn interrupt_turn(&self, thread_id: ThreadId, turn_id: TurnId) -> anyhow::Result<()> {
3286        self.interrupt_turn_with_reason(thread_id, turn_id, TurnLifecycleReason::UserInterrupt)
3287            .await
3288    }
3289
3290    async fn interrupt_turn_with_reason(
3291        &self,
3292        thread_id: ThreadId,
3293        turn_id: TurnId,
3294        reason: TurnLifecycleReason,
3295    ) -> anyhow::Result<()> {
3296        let handle = self.active_turns.write().await.remove(&turn_id);
3297        if let Some(handle) = handle {
3298            if handle
3299                .drain
3300                .interrupt_requested
3301                .swap(true, Ordering::AcqRel)
3302            {
3303                return Ok(());
3304            }
3305            *handle.drain.interrupt_reason.lock().await = Some(reason);
3306            self.turn_drains
3307                .write()
3308                .await
3309                .insert(turn_id.clone(), handle.drain.clone());
3310            let ownership = self.provider_cleanup_ownership(&turn_id);
3311            self.record_turn_lifecycle_with_ownership(
3312                thread_id.clone(),
3313                turn_id.clone(),
3314                TurnLifecycleState::InterruptRequested,
3315                TurnCleanupState::Requested,
3316                Some(reason),
3317                ownership,
3318            )
3319            .await;
3320            handle.abort.abort();
3321            self.sync_turn_goal_usage(&thread_id, &turn_id).await?;
3322            if reason == TurnLifecycleReason::UserInterrupt {
3323                self.goals
3324                    .finish_turn(
3325                        &thread_id,
3326                        &turn_id,
3327                        Some(roder_api::goals::ThreadGoalStatus::Paused),
3328                    )
3329                    .await?;
3330            }
3331            self.active_turns_changed.notify_waiters();
3332        }
3333        self.active_turn_selections.write().await.remove(&turn_id);
3334        self.cancel_pending_external_tool_calls_for_turn(&turn_id)
3335            .await;
3336        Ok(())
3337    }
3338
3339    pub async fn steer_turn(
3340        &self,
3341        thread_id: ThreadId,
3342        turn_id: TurnId,
3343        message: String,
3344        images: Vec<InputImage>,
3345    ) -> anyhow::Result<()> {
3346        self.enqueue_turn_steer(thread_id, turn_id, message, images, None)
3347            .await
3348    }
3349
3350    pub(crate) async fn has_pending_turn_steers(&self, turn_id: &TurnId) -> bool {
3351        let active = self.active_turns.read().await.get(turn_id).cloned();
3352        let Some(active) = active else {
3353            return false;
3354        };
3355        let has_pending = !active.steers.lock().await.is_empty();
3356        has_pending
3357    }
3358
3359    async fn steer_turn_with_mailbox_ack(
3360        &self,
3361        thread_id: ThreadId,
3362        turn_id: TurnId,
3363        message: String,
3364        message_ids: Vec<String>,
3365        team_id: TeamId,
3366    ) -> anyhow::Result<()> {
3367        self.enqueue_turn_steer(
3368            thread_id,
3369            turn_id,
3370            message,
3371            Vec::new(),
3372            Some(MailboxDeliveryAck {
3373                team_id,
3374                message_ids,
3375            }),
3376        )
3377        .await
3378    }
3379
3380    async fn deliver_pending_team_mailbox(
3381        &self,
3382        team_id: &str,
3383        member_id: &str,
3384        member_thread_id: ThreadId,
3385        turn_id: TurnId,
3386    ) -> anyhow::Result<()> {
3387        let pending = self
3388            .teams
3389            .reserve_pending_mailbox_messages(team_id, member_id, &turn_id)
3390            .await?;
3391        if pending.is_empty() {
3392            return Ok(());
3393        }
3394        let message_ids = pending
3395            .iter()
3396            .map(|message| message.id.clone())
3397            .collect::<Vec<_>>();
3398        let team = self
3399            .read_team(team_id)
3400            .await
3401            .ok_or_else(|| anyhow::anyhow!("unknown team {team_id:?}"))?;
3402        if let Err(error) = self
3403            .steer_turn_with_mailbox_ack(
3404                member_thread_id,
3405                turn_id.clone(),
3406                format_mailbox_messages(&team, &pending),
3407                message_ids.clone(),
3408                team_id.to_string(),
3409            )
3410            .await
3411        {
3412            self.teams
3413                .release_mailbox_reservations(&turn_id, &message_ids)
3414                .await;
3415            return Err(error);
3416        }
3417        Ok(())
3418    }
3419
3420    async fn enqueue_turn_steer(
3421        &self,
3422        thread_id: ThreadId,
3423        turn_id: TurnId,
3424        message: String,
3425        images: Vec<InputImage>,
3426        mailbox_ack: Option<MailboxDeliveryAck>,
3427    ) -> anyhow::Result<()> {
3428        let message = message.trim().to_string();
3429        if message.is_empty() && images.is_empty() {
3430            return Ok(());
3431        }
3432
3433        let Some(active) = self.active_turns.read().await.get(&turn_id).cloned() else {
3434            anyhow::bail!("no active turn to steer");
3435        };
3436        anyhow::ensure!(
3437            active.thread_id == thread_id,
3438            "turn does not belong to thread"
3439        );
3440        {
3441            let mut steers = active.steers.lock().await;
3442            steers.push(QueuedTurnSteer {
3443                message: UserMessage::with_images(message.clone(), images),
3444                mailbox_ack,
3445            });
3446            active
3447                .steer_changed
3448                .send_modify(|generation| *generation = generation.wrapping_add(1));
3449        }
3450        self.emit(RoderEvent::TurnSteered(TurnSteered {
3451            thread_id,
3452            turn_id,
3453            message,
3454            timestamp: OffsetDateTime::now_utc(),
3455        }))
3456        .await;
3457        Ok(())
3458    }
3459
3460    pub async fn tool_specs(&self) -> Vec<roder_api::tools::ToolSpec> {
3461        let cfg = self.config.read().await;
3462        let model_profile =
3463            model_profile_for_provider_model(&cfg, &cfg.default_provider, &cfg.default_model);
3464        self.filtered_tool_specs(
3465            &cfg,
3466            &cfg.default_model,
3467            &cfg.default_provider,
3468            model_profile.as_ref(),
3469            &[],
3470            &[],
3471        )
3472    }
3473
3474    pub fn subagent_definitions(&self) -> Vec<SubagentDefinition> {
3475        self.registry
3476            .subagent_dispatchers
3477            .iter()
3478            .flat_map(|dispatcher| dispatcher.definitions())
3479            .collect()
3480    }
3481
3482    async fn run_turn(
3483        self: &Arc<Self>,
3484        req: StartTurnRequest,
3485        turn_id: TurnId,
3486        initial_mailbox_ack: Option<MailboxDeliveryAck>,
3487        mut steering: tokio::sync::watch::Receiver<u64>,
3488        automatic_goal_turn: bool,
3489    ) -> anyhow::Result<TurnRunOutcome> {
3490        self.ensure_execution_authority()?;
3491        self.goals
3492            .begin_turn(&req.thread_id, &turn_id, automatic_goal_turn)
3493            .await?;
3494        let turn_started_at = OffsetDateTime::now_utc();
3495        self.emit(RoderEvent::TurnStarted(TurnStarted {
3496            thread_id: req.thread_id.clone(),
3497            turn_id: turn_id.clone(),
3498            runtime_profile: self.config.read().await.runtime_profile,
3499            timestamp: turn_started_at,
3500        }))
3501        .await;
3502        self.persist_turn_item(
3503            &req.thread_id,
3504            &turn_id,
3505            &TranscriptItem::UserMessage(UserMessage::with_images(
3506                req.message.clone(),
3507                req.images.clone(),
3508            )),
3509        )
3510        .await?;
3511        crate::hooks::run_lifecycle(
3512            self,
3513            &req.thread_id,
3514            &turn_id,
3515            Some(&req.workspace),
3516            "UserPromptSubmit",
3517            None,
3518            serde_json::json!({"prompt": req.message}),
3519        )
3520        .await;
3521        if let Some(ack) = initial_mailbox_ack {
3522            self.teams
3523                .mark_mailbox_messages_delivered(&ack.team_id, &turn_id, &ack.message_ids)
3524                .await?;
3525        }
3526
3527        let mut cfg = self.config.read().await.clone();
3528        let runtime_profile = cfg.runtime_profile;
3529        let turn_deadline = turn_deadline_for_config(&cfg);
3530        let deadline_finalization_reserve =
3531            crate::deadline_policy::finalization_reserve_seconds(cfg.turn_deadline_seconds);
3532        let selection_mode = self.selection_mode_for_thread(&req.thread_id).await?;
3533        let concrete_selection = selection_mode
3534            .as_ref()
3535            .map(ModelSelectionMode::concrete_selection);
3536        let default_provider = req
3537            .provider_override
3538            .clone()
3539            .or_else(|| {
3540                concrete_selection
3541                    .as_ref()
3542                    .map(|selection| selection.provider.clone())
3543            })
3544            .unwrap_or(cfg.default_provider.clone());
3545        let default_model = req
3546            .model_override
3547            .clone()
3548            .or_else(|| {
3549                concrete_selection
3550                    .as_ref()
3551                    .map(|selection| selection.model.clone())
3552            })
3553            .unwrap_or(cfg.default_model.clone());
3554        if let Some(reasoning) = req.reasoning_override.as_deref().or_else(|| {
3555            selection_mode
3556                .as_ref()
3557                .and_then(ModelSelectionMode::reasoning)
3558        }) {
3559            validate_reasoning_effort(&default_model, reasoning)?;
3560            cfg.reasoning = Some(reasoning.to_string());
3561        }
3562        let turn_has_concrete_model_override =
3563            req.provider_override.is_some() || req.model_override.is_some();
3564        let (turn_inference_router, turn_inference_router_profile) = match &selection_mode {
3565            Some(ModelSelectionMode::Auto {
3566                router_id, profile, ..
3567            }) if !turn_has_concrete_model_override => (
3568                RuntimeInferenceRouterConfig {
3569                    enabled: true,
3570                    router_id: Some(router_id.clone()),
3571                },
3572                profile.clone(),
3573            ),
3574            _ => (RuntimeInferenceRouterConfig::disabled(), None),
3575        };
3576        let mut provider = default_provider.clone();
3577        let mut model = default_model.clone();
3578        let mut model_profile = model_profile_for_provider_model(&cfg, &provider, &model);
3579        let workspace = req.workspace.clone();
3580        let compaction_generation_at_start = self.compaction_generation(&req.thread_id);
3581        let mut transcript = self.transcript_for_turn(&req, &turn_id).await?;
3582        let mut compacted_this_turn =
3583            self.compaction_generation(&req.thread_id) != compaction_generation_at_start;
3584        let agent_swarm_mode_active = self
3585            .effective_agent_swarm_mode_for_thread(&req.thread_id)
3586            .await;
3587        let ultra_mode_active = self.effective_ultra_mode_for_thread(&req.thread_id).await;
3588        let thread_overrides = self.thread_turn_overrides(&req.thread_id).await?;
3589        let mut final_assistant_text = String::new();
3590        let mut final_provider_metadata = None;
3591        let mut exhausted_tool_rounds = true;
3592        let mut verification_gate =
3593            VerificationGateState::new(req.message.clone(), runtime_profile);
3594        let mut speed_policy = SpeedPolicyState::default();
3595        let mut reliability = TurnReliabilityState::default();
3596        // Overwritten on every inference step's Completed event so only the
3597        // terminal step's stop reason survives (mid-turn tool_use steps must
3598        // not leak onto turn/completed).
3599        let mut turn_finish_reason: Option<String> = None;
3600        let mut deadline_finalization_requested = false;
3601        let mut deadline_scoreable_completion_requested = false;
3602        let mut task_ledger_completion_reminders = 0_u8;
3603        let mut task_ledger_scoreable_checkpoints = 0_u8;
3604        let mut empty_tool_call_nudges_used = 0_u32;
3605        let mut provider_stream_retry_attempts = 0_u32;
3606        let mut context_limit_recovery_attempts = 0_u32;
3607        let mut routing_candidates = None;
3608        let routing_transcript_start = transcript.len().saturating_sub(1);
3609        let mut routing_escalations = 0_u32;
3610        let mut model_switch_summary_selection = None::<ModelSelection>;
3611
3612        'tool_rounds: for round_index in 0..MAX_TOOL_ROUNDS_PER_TURN {
3613            if let Some(deadline) = turn_deadline
3614                && deadline_expired(deadline)
3615            {
3616                self.fail_turn_due_to_deadline(&req.thread_id, &turn_id, deadline, &transcript)
3617                    .await?;
3618                return Ok(TurnRunOutcome::Stopped);
3619            }
3620            let steers = self.drain_turn_steers(&turn_id, &mut steering).await;
3621            self.append_steers(&req, &turn_id, &mut transcript, steers)
3622                .await?;
3623            if runtime_profile == RuntimeProfile::Eval
3624                && let Some(remaining) = crate::deadline_policy::should_start_finalization(
3625                    turn_deadline,
3626                    deadline_finalization_reserve,
3627                    deadline_finalization_requested || deadline_scoreable_completion_requested,
3628                )
3629            {
3630                if req.task_ledger_required
3631                    && task_ledger_completion_reminders < TASK_LEDGER_COMPLETION_REMINDER_LIMIT
3632                    && let Some(prompt) = task_ledger_completion_prompt(&transcript)
3633                {
3634                    task_ledger_completion_reminders += 1;
3635                    deadline_scoreable_completion_requested = true;
3636                    let item = TranscriptItem::UserMessage(UserMessage::text(
3637                        task_ledger_deadline_completion_prompt(
3638                            remaining,
3639                            deadline_finalization_reserve,
3640                            &prompt,
3641                        ),
3642                    ));
3643                    self.persist_turn_item(&req.thread_id, &turn_id, &item)
3644                        .await?;
3645                    transcript.push(item);
3646                    continue 'tool_rounds;
3647                } else {
3648                    self.start_deadline_finalization(
3649                        &req.thread_id,
3650                        &turn_id,
3651                        &mut transcript,
3652                        remaining,
3653                    )
3654                    .await?;
3655                    deadline_finalization_requested = true;
3656                }
3657            }
3658            if runtime_profile == RuntimeProfile::Eval
3659                && req.task_ledger_required
3660                && !deadline_finalization_requested
3661                && task_ledger_scoreable_checkpoints < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
3662                && let Some(remaining) = deadline_remaining_seconds(turn_deadline)
3663                && remaining <= TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS
3664                && remaining > deadline_finalization_reserve
3665                && let Some(prompt) = task_ledger_completion_prompt(&transcript)
3666            {
3667                task_ledger_scoreable_checkpoints += 1;
3668                let item = TranscriptItem::UserMessage(UserMessage::text(
3669                    task_ledger_scoreable_checkpoint_prompt(remaining, &prompt),
3670                ));
3671                self.persist_turn_item(&req.thread_id, &turn_id, &item)
3672                    .await?;
3673                transcript.push(item);
3674                continue 'tool_rounds;
3675            }
3676            if turn_inference_router.is_active() && routing_candidates.is_none() {
3677                routing_candidates =
3678                    Some(collect_inference_routing_candidates(&self.registry).await);
3679            }
3680            let routing_tools_model = model.clone();
3681            let routing_tools_provider = provider.clone();
3682            let routing_tools = self.filtered_tool_specs(
3683                &cfg,
3684                &model,
3685                &provider,
3686                model_profile.as_ref(),
3687                &thread_overrides.tool_allowlist,
3688                &thread_overrides.external_tools,
3689            );
3690            let prior_failures =
3691                transcript_failure_count_since(&transcript, routing_transcript_start)
3692                    .max(reliability.tool_failure_count())
3693                    .saturating_add(provider_stream_retry_attempts);
3694            let routing_selection = route_inference_selection(
3695                &self.registry,
3696                &turn_inference_router,
3697                InferenceRoutingRequest {
3698                    thread_id: &req.thread_id,
3699                    turn_id: &turn_id,
3700                    round_index: round_index as u32,
3701                    runtime_profile,
3702                    phase: speed_policy.phase(),
3703                    profile: turn_inference_router_profile.as_deref(),
3704                    default_selection: ModelSelection {
3705                        provider: default_provider.clone(),
3706                        model: default_model.clone(),
3707                    },
3708                    transcript: &transcript,
3709                    tools: &routing_tools,
3710                    candidates: routing_candidates.as_deref(),
3711                    prior_failures,
3712                    prior_escalations: routing_escalations,
3713                },
3714            )
3715            .await;
3716            if let Some(decision) = routing_selection.decision.clone() {
3717                if matches!(decision.outcome, InferenceRoutingOutcome::Escalated) {
3718                    routing_escalations = routing_escalations.saturating_add(1);
3719                }
3720                self.emit(RoderEvent::InferenceRoutingDecision(
3721                    InferenceRoutingDecisionEvent {
3722                        thread_id: req.thread_id.clone(),
3723                        turn_id: turn_id.clone(),
3724                        round_index: round_index as u32,
3725                        default_selection: ModelSelection {
3726                            provider: default_provider.clone(),
3727                            model: default_model.clone(),
3728                        },
3729                        selected_selection: routing_selection.selection.clone(),
3730                        decision,
3731                        timestamp: OffsetDateTime::now_utc(),
3732                    },
3733                ))
3734                .await;
3735            }
3736            provider = routing_selection.selection.provider.clone();
3737            model = routing_selection.selection.model.clone();
3738            let engine = self.engine_for(&provider)?;
3739            let capabilities = engine.capabilities();
3740            let tool_images = ToolImages::for_engine(engine.as_ref(), &model);
3741            model_profile = model_profile_for_provider_model(&cfg, &provider, &model);
3742            let tools = if capabilities.tool_calls {
3743                if model == routing_tools_model && provider == routing_tools_provider {
3744                    routing_tools.clone()
3745                } else {
3746                    self.filtered_tool_specs(
3747                        &cfg,
3748                        &model,
3749                        &provider,
3750                        model_profile.as_ref(),
3751                        &thread_overrides.tool_allowlist,
3752                        &thread_overrides.external_tools,
3753                    )
3754                }
3755            } else {
3756                Vec::new()
3757            };
3758            let parallel_tool_calls = parallel_tool_calls_for_model(&cfg, &model);
3759            let tool_choice = if tools.is_empty() {
3760                ToolChoice::None
3761            } else {
3762                ToolChoice::Auto
3763            };
3764            let summary_selection = ModelSelection {
3765                provider: provider.clone(),
3766                model: model.clone(),
3767            };
3768            if model_switch_summary_selection.as_ref() != Some(&summary_selection) {
3769                if let Some(summary) = model_switch_summary(
3770                    &transcript,
3771                    model_profile.as_ref(),
3772                    &provider,
3773                    &model,
3774                    &tools,
3775                ) {
3776                    let item = TranscriptItem::UserMessage(UserMessage::text(summary));
3777                    self.persist_turn_item(&req.thread_id, &turn_id, &item)
3778                        .await?;
3779                    transcript.push(item);
3780                }
3781                model_switch_summary_selection = Some(summary_selection);
3782            }
3783
3784            if !capabilities.image_input && transcript_has_images(&transcript) {
3785                self.fail_turn_with_error(
3786                    &req.thread_id,
3787                    &turn_id,
3788                    format!("provider {provider} does not support image input"),
3789                )
3790                .await?;
3791                return Ok(TurnRunOutcome::Stopped);
3792            }
3793            let compaction_generation_before = self.compaction_generation(&req.thread_id);
3794            transcript = self
3795                .compact_transcript_if_needed(
3796                    &req.thread_id,
3797                    &turn_id,
3798                    &provider,
3799                    &model,
3800                    transcript,
3801                    self.compaction_options_for_turn(&req.thread_id, !compacted_this_turn),
3802                )
3803                .await?;
3804            compacted_this_turn |=
3805                self.compaction_generation(&req.thread_id) != compaction_generation_before;
3806            if round_index == 0 {
3807                self.complete_context_assembly(&req, &turn_id, &transcript, tool_images)
3808                    .await;
3809            }
3810
3811            let speed_policy_decision =
3812                speed_policy.decision(runtime_profile, &model, &cfg.speed_policy);
3813            let request_reasoning = reasoning_from_decision(
3814                speed_policy_decision.as_ref(),
3815                routing_selection
3816                    .reasoning
3817                    .clone()
3818                    .unwrap_or_else(|| reasoning_for_model(&cfg, &model)),
3819            );
3820            self.active_turn_selections.write().await.insert(
3821                turn_id.clone(),
3822                ModelSelectionMode::manual(
3823                    provider.clone(),
3824                    model.clone(),
3825                    request_reasoning.level.clone(),
3826                ),
3827            );
3828            if let Some(limit) = reliability.record_model_call(
3829                &cfg.reliability,
3830                runtime_profile == RuntimeProfile::Interactive,
3831            ) {
3832                self.fail_turn_due_to_reliability_limit(
3833                    &req.thread_id,
3834                    &turn_id,
3835                    &provider,
3836                    &model,
3837                    limit,
3838                    &transcript,
3839                )
3840                .await?;
3841                return Ok(TurnRunOutcome::Stopped);
3842            }
3843            self.emit(RoderEvent::InferenceStarted(InferenceStarted {
3844                thread_id: req.thread_id.clone(),
3845                turn_id: turn_id.clone(),
3846                engine_id: engine.id(),
3847                model: ModelSelection {
3848                    provider: provider.clone(),
3849                    model: model.clone(),
3850                },
3851                reasoning: request_reasoning.clone(),
3852                speed_policy: speed_policy_decision.clone(),
3853                deadline_remaining_seconds: deadline_remaining_seconds(turn_deadline),
3854                timestamp: OffsetDateTime::now_utc(),
3855            }))
3856            .await;
3857
3858            let mut instructions = req.instructions.clone();
3859            if let Some(extra) = &thread_overrides.developer_instructions {
3860                instructions = apply_thread_developer_instructions(instructions, extra);
3861            }
3862            if let Some(context) = req.developer_context.as_deref() {
3863                instructions = apply_turn_developer_context(instructions, context);
3864            }
3865            let mut instructions = apply_runtime_profile(instructions, runtime_profile);
3866            if let Some(profile) = &model_profile {
3867                instructions = apply_model_instruction_overlay(instructions, profile);
3868            }
3869            if req.task_ledger_required
3870                && runtime_profile == RuntimeProfile::Eval
3871                && !transcript_has_task_ledger(&transcript)
3872            {
3873                instructions = apply_task_ledger_required(instructions);
3874            }
3875            let effective_policy_mode = self.effective_policy_mode_for_thread(&req.thread_id).await;
3876            if effective_policy_mode == PolicyMode::Plan {
3877                instructions = apply_plan_mode(instructions);
3878            }
3879            if agent_swarm_mode_active {
3880                instructions = apply_agent_swarm_mode(instructions);
3881            }
3882            // Ultra mode (any model) enables proactive multi-agent policy.
3883            // Codex Sol/Terra Ultra effort still does too without requiring the
3884            // mode flag. Models that advertise Ultra effort keep the
3885            // explicit-request-only policy on lower efforts when ultra mode is
3886            // off; other models get multi-agent policy only when ultra mode is on.
3887            let model_has_ultra_effort = model_supports_reasoning_effort(&model, REASONING_ULTRA);
3888            let effort_is_ultra = request_reasoning.level.as_deref() == Some(REASONING_ULTRA);
3889            let proactive_multi_agent = ultra_mode_active || effort_is_ultra;
3890            if ultra_mode_active || model_has_ultra_effort {
3891                instructions = apply_codex_multi_agent_mode(instructions, proactive_multi_agent);
3892            }
3893            let compaction_instructions = instructions.clone();
3894            instructions = self
3895                .goals
3896                .apply_goal_instructions(&req.thread_id, instructions, effective_policy_mode)
3897                .await?;
3898            let mut request_metadata = serde_json::json!({});
3899            if let Some(decision) = &speed_policy_decision {
3900                request_metadata["speedPolicy"] = serde_json::json!(decision);
3901            }
3902            if let Some(decision) = routing_selection.decision.as_ref() {
3903                request_metadata["inferenceRouting"] = serde_json::json!(decision);
3904            }
3905            if let Some(remaining) = deadline_remaining_seconds(turn_deadline) {
3906                request_metadata["deadlineRemainingSeconds"] = serde_json::json!(remaining);
3907            }
3908            if let Some(profile) = &model_profile {
3909                request_metadata["modelProfile"] = serde_json::json!({
3910                    "model": profile.model,
3911                    "providerFamily": profile.provider_family,
3912                    "editTool": profile.edit_tool,
3913                    "schemaPolicy": profile.schema_policy,
3914                    "instructionOverlay": profile.instruction_overlay,
3915                    "parallelToolCalls": profile.parallel_tool_calls,
3916                    "autoCompactTokenLimit": profile.auto_compact_token_limit,
3917                });
3918            }
3919            let task_ledger_required_this_round = req.task_ledger_required
3920                && runtime_profile == RuntimeProfile::Eval
3921                && !deadline_finalization_requested
3922                && !transcript_has_task_ledger(&transcript);
3923            let task_ledger_tools = (capabilities.tool_calls && task_ledger_required_this_round)
3924                .then(|| self.task_ledger_tool_specs(model_profile.as_ref()))
3925                .filter(|tools| !tools.is_empty());
3926            let request_tools = if deadline_finalization_requested {
3927                Vec::new()
3928            } else if let Some(ledger_tools) = &task_ledger_tools {
3929                ledger_tools.clone()
3930            } else {
3931                tools.clone()
3932            };
3933            // Tool guidance describes the tools this request can call, which is
3934            // fewer than `tools` on a deadline finalization or forced ledger round.
3935            instructions = apply_parallel_web_tools(
3936                instructions,
3937                request_tools.iter().map(|spec| spec.name.as_str()),
3938            );
3939            instructions = apply_browser_routing(
3940                instructions,
3941                request_tools.iter().map(|spec| spec.name.as_str()),
3942            );
3943            let request_tool_choice = if deadline_finalization_requested {
3944                ToolChoice::None
3945            } else if task_ledger_tools.is_some() {
3946                ToolChoice::Specific(TASK_LEDGER_TOOL_NAME.to_string())
3947            } else {
3948                tool_choice.clone()
3949            };
3950            if deadline_finalization_requested {
3951                request_metadata["deadlineFinalization"] = serde_json::json!({
3952                    "reserveSeconds": deadline_finalization_reserve,
3953                    "remainingSeconds": deadline_remaining_seconds(turn_deadline),
3954                });
3955            } else if deadline_scoreable_completion_requested {
3956                request_metadata["deadlineScoreableCompletion"] = serde_json::json!({
3957                    "reserveSeconds": deadline_finalization_reserve,
3958                    "remainingSeconds": deadline_remaining_seconds(turn_deadline),
3959                });
3960            }
3961            let mut eager_tools = EagerTools::new(
3962                self,
3963                &request_tools,
3964                &thread_overrides.external_tools,
3965                parallel_tool_calls,
3966            );
3967            let mut request_reliability: ReliabilityRequestPolicy = cfg.reliability.clone().into();
3968            request_reliability.provider_retry_max_attempts = request_reliability
3969                .provider_retry_max_attempts
3970                .saturating_sub(provider_stream_retry_attempts)
3971                .max(1);
3972            let request = AgentInferenceRequest {
3973                model: ModelSelection {
3974                    provider: provider.clone(),
3975                    model: model.clone(),
3976                },
3977                instructions,
3978                transcript: crate::tool_result_images::request_transcript(&transcript, tool_images),
3979                tools: request_tools,
3980                tool_choice: request_tool_choice,
3981                reasoning: request_reasoning,
3982                output: OutputConfig::default(),
3983                runtime: RuntimeHints {
3984                    auto_compact_token_limit: (provider != roder_api::catalog::PROVIDER_CODEX)
3985                        .then(|| server_side_compaction_threshold(&cfg, &model))
3986                        .flatten(),
3987                    profile: runtime_profile,
3988                    parallel_tool_calls: Some(parallel_tool_calls),
3989                    prompt_cache_key: Some(req.thread_id.clone()),
3990                    hosted_web_search: cfg.hosted_web_search.clone(),
3991                    tool_search: tool_search_for_provider_model(&cfg, &provider, &model),
3992                    speed_policy: speed_policy_decision,
3993                    reliability: Some(request_reliability),
3994                    deadline_remaining_seconds: deadline_remaining_seconds(turn_deadline),
3995                    service_tier: req
3996                        .service_tier_override
3997                        .clone()
3998                        .or_else(|| thread_overrides.service_tier.clone()),
3999                    ..RuntimeHints::default()
4000                },
4001                metadata: request_metadata,
4002            };
4003
4004            let mut compaction_template = request.clone();
4005            compaction_template.instructions = compaction_instructions;
4006            compaction_template.transcript.clear();
4007            self.compaction_templates
4008                .write()
4009                .await
4010                .insert(req.thread_id.clone(), compaction_template);
4011
4012            let ctx = InferenceTurnContext {
4013                thread_id: &req.thread_id,
4014                turn_id: &turn_id,
4015                tool_executor: Some(std::sync::Arc::new(
4016                    crate::tool_execution::RuntimeTurnToolExecutor {
4017                        runtime: Arc::clone(self),
4018                        thread_id: req.thread_id.clone(),
4019                        turn_id: turn_id.clone(),
4020                        workspace: Some(workspace.clone()),
4021                        deadline: turn_deadline,
4022                    },
4023                )),
4024            };
4025            let stream_future = preemptible(engine.stream_turn(ctx, request), &mut steering);
4026            let mut stream = if let Some((deadline, timeout_action)) = inference_timeout_deadline(
4027                turn_deadline,
4028                runtime_profile,
4029                req.task_ledger_required,
4030                deadline_finalization_reserve,
4031                deadline_finalization_requested || deadline_scoreable_completion_requested,
4032                task_ledger_scoreable_checkpoints,
4033                &transcript,
4034            ) {
4035                match tokio::time::timeout_at(deadline_instant(deadline), stream_future).await {
4036                    Ok(stream) => match stream {
4037                        Ok(stream) => stream,
4038                        Err(err) => {
4039                            if err.is::<SamplingPreempted>() {
4040                                self.finish_sampling_cleanup(&turn_id).await?;
4041                                provider_stream_retry_attempts = 0;
4042                                continue 'tool_rounds;
4043                            }
4044                            self.finish_sampling_cleanup(&turn_id).await?;
4045                            if !deadline_finalization_requested
4046                                && self
4047                                    .retry_sampling_failure(
4048                                        SamplingRetry {
4049                                            thread_id: &req.thread_id,
4050                                            turn_id: &turn_id,
4051                                            provider: &provider,
4052                                            model: &model,
4053                                            config: &cfg.reliability,
4054                                            error: &err,
4055                                        },
4056                                        &mut provider_stream_retry_attempts,
4057                                        &mut steering,
4058                                    )
4059                                    .await
4060                            {
4061                                continue 'tool_rounds;
4062                            }
4063                            let error = err.to_string();
4064                            if self
4065                                .try_recover_from_context_limit(
4066                                    &req.thread_id,
4067                                    &turn_id,
4068                                    &provider,
4069                                    &model,
4070                                    &error,
4071                                    crate::compaction::is_context_overflow_failure(&err),
4072                                    &mut transcript,
4073                                    &mut compacted_this_turn,
4074                                    &mut context_limit_recovery_attempts,
4075                                )
4076                                .await?
4077                            {
4078                                continue 'tool_rounds;
4079                            }
4080                            return Err(err);
4081                        }
4082                    },
4083                    Err(_) => {
4084                        if runtime_profile == RuntimeProfile::Eval
4085                            && !deadline_finalization_requested
4086                        {
4087                            let remaining = deadline_remaining_seconds(turn_deadline).unwrap_or(0);
4088                            if timeout_action == InferenceTimeoutAction::ScoreableCheckpoint
4089                                && task_ledger_scoreable_checkpoints
4090                                    < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
4091                                && let Some(prompt) = task_ledger_completion_prompt(&transcript)
4092                            {
4093                                task_ledger_scoreable_checkpoints += 1;
4094                                let item = TranscriptItem::UserMessage(UserMessage::text(
4095                                    task_ledger_scoreable_checkpoint_prompt(remaining, &prompt),
4096                                ));
4097                                self.persist_turn_item(&req.thread_id, &turn_id, &item)
4098                                    .await?;
4099                                transcript.push(item);
4100                                continue 'tool_rounds;
4101                            }
4102                            self.start_deadline_finalization(
4103                                &req.thread_id,
4104                                &turn_id,
4105                                &mut transcript,
4106                                remaining,
4107                            )
4108                            .await?;
4109                            deadline_finalization_requested = true;
4110                            continue 'tool_rounds;
4111                        }
4112                        self.fail_turn_due_to_deadline(
4113                            &req.thread_id,
4114                            &turn_id,
4115                            deadline,
4116                            &transcript,
4117                        )
4118                        .await?;
4119                        return Ok(TurnRunOutcome::Stopped);
4120                    }
4121                }
4122            } else {
4123                match stream_future.await {
4124                    Ok(stream) => stream,
4125                    Err(err) => {
4126                        if err.is::<SamplingPreempted>() {
4127                            self.finish_sampling_cleanup(&turn_id).await?;
4128                            provider_stream_retry_attempts = 0;
4129                            continue 'tool_rounds;
4130                        }
4131                        self.finish_sampling_cleanup(&turn_id).await?;
4132                        if !deadline_finalization_requested
4133                            && self
4134                                .retry_sampling_failure(
4135                                    SamplingRetry {
4136                                        thread_id: &req.thread_id,
4137                                        turn_id: &turn_id,
4138                                        provider: &provider,
4139                                        model: &model,
4140                                        config: &cfg.reliability,
4141                                        error: &err,
4142                                    },
4143                                    &mut provider_stream_retry_attempts,
4144                                    &mut steering,
4145                                )
4146                                .await
4147                        {
4148                            continue 'tool_rounds;
4149                        }
4150                        let error = err.to_string();
4151                        if self
4152                            .try_recover_from_context_limit(
4153                                &req.thread_id,
4154                                &turn_id,
4155                                &provider,
4156                                &model,
4157                                &error,
4158                                crate::compaction::is_context_overflow_failure(&err),
4159                                &mut transcript,
4160                                &mut compacted_this_turn,
4161                                &mut context_limit_recovery_attempts,
4162                            )
4163                            .await?
4164                        {
4165                            continue 'tool_rounds;
4166                        }
4167                        return Err(err);
4168                    }
4169                }
4170            };
4171            let mut sampling_output = SamplingOutput::default();
4172            let output_context = OutputContext {
4173                thread_id: &req.thread_id,
4174                turn_id: &turn_id,
4175                profile: model_profile.as_ref(),
4176                provider: &provider,
4177                model: &model,
4178            };
4179            let mut assistant_text = String::new();
4180            let mut phase_messages = Vec::<AssistantMessage>::new();
4181            let mut reasoning_text = String::new();
4182            let mut replayed_tool_call = false;
4183            let mut tool_calls = Vec::new();
4184            let mut patch_progress = PatchProgressTracker::default();
4185            let mut provider_metadata = None;
4186            let mut provider_requests_continuation = false;
4187            // Captured from mid-stream Compaction(completed) events. Codex/OpenAI
4188            // sometimes abort the SSE stream after emitting the compaction item
4189            // ("error decoding response body") before response.completed, so we
4190            // must not rely solely on ProviderMetadata for the boundary.
4191            let mut stream_compaction_item: Option<serde_json::Value> = None;
4192
4193            loop {
4194                let next_future = async {
4195                    loop {
4196                        tokio::select! {
4197                            biased;
4198                            _ = wait_for_steer(&mut steering) => break Some(Err(SamplingPreempted.into())),
4199                            result = eager_tools.poll_result(), if eager_tools.pending() => {
4200                                if let Err(error) = result { break Some(Err(error)); }
4201                            }
4202                            next = stream.next() => break next,
4203                        }
4204                    }
4205                };
4206                let next = if let Some((deadline, timeout_action)) = inference_timeout_deadline(
4207                    turn_deadline,
4208                    runtime_profile,
4209                    req.task_ledger_required,
4210                    deadline_finalization_reserve,
4211                    deadline_finalization_requested || deadline_scoreable_completion_requested,
4212                    task_ledger_scoreable_checkpoints,
4213                    &transcript,
4214                ) {
4215                    match tokio::time::timeout_at(deadline_instant(deadline), next_future).await {
4216                        Ok(next) => next,
4217                        Err(_) => {
4218                            if runtime_profile == RuntimeProfile::Eval
4219                                && !deadline_finalization_requested
4220                            {
4221                                let remaining =
4222                                    deadline_remaining_seconds(turn_deadline).unwrap_or(0);
4223                                if timeout_action == InferenceTimeoutAction::ScoreableCheckpoint
4224                                    && task_ledger_scoreable_checkpoints
4225                                        < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
4226                                    && let Some(prompt) = task_ledger_completion_prompt(&transcript)
4227                                {
4228                                    task_ledger_scoreable_checkpoints += 1;
4229                                    let item = TranscriptItem::UserMessage(UserMessage::text(
4230                                        task_ledger_scoreable_checkpoint_prompt(remaining, &prompt),
4231                                    ));
4232                                    self.persist_turn_item(&req.thread_id, &turn_id, &item)
4233                                        .await?;
4234                                    transcript.push(item);
4235                                    continue 'tool_rounds;
4236                                }
4237                                self.start_deadline_finalization(
4238                                    &req.thread_id,
4239                                    &turn_id,
4240                                    &mut transcript,
4241                                    remaining,
4242                                )
4243                                .await?;
4244                                deadline_finalization_requested = true;
4245                                continue 'tool_rounds;
4246                            }
4247                            self.fail_turn_due_to_deadline(
4248                                &req.thread_id,
4249                                &turn_id,
4250                                deadline,
4251                                &transcript,
4252                            )
4253                            .await?;
4254                            return Ok(TurnRunOutcome::Stopped);
4255                        }
4256                    }
4257                } else {
4258                    next_future.await
4259                };
4260                let res = next.unwrap_or_else(|| {
4261                    Err(roder_api::provider_error::ProviderFailure::new(
4262                        roder_api::provider_error::ProviderFailureKind::StreamInterrupted,
4263                        "provider stream ended before terminal completion",
4264                    )
4265                    .into())
4266                });
4267                let event = match res {
4268                    Ok(event) => event,
4269                    Err(err) => {
4270                        drop(stream);
4271                        eager_tools.drain().await?;
4272                        for result in eager_tools.take_results(&tool_calls) {
4273                            verification_gate.record_tool_result(&result);
4274                            transcript.push(TranscriptItem::ToolResult(result));
4275                        }
4276                        self.finish_sampling_cleanup(&turn_id).await?;
4277                        if err.is::<SamplingPreempted>() {
4278                            provider_stream_retry_attempts = 0;
4279                            continue 'tool_rounds;
4280                        }
4281                        if !deadline_finalization_requested
4282                            && self
4283                                .retry_sampling_failure(
4284                                    SamplingRetry {
4285                                        thread_id: &req.thread_id,
4286                                        turn_id: &turn_id,
4287                                        provider: &provider,
4288                                        model: &model,
4289                                        config: &cfg.reliability,
4290                                        error: &err,
4291                                    },
4292                                    &mut provider_stream_retry_attempts,
4293                                    &mut steering,
4294                                )
4295                                .await
4296                        {
4297                            continue 'tool_rounds;
4298                        }
4299                        let error = err.to_string();
4300                        if self
4301                            .try_recover_from_context_limit(
4302                                &req.thread_id,
4303                                &turn_id,
4304                                &provider,
4305                                &model,
4306                                &error,
4307                                crate::compaction::is_context_overflow_failure(&err),
4308                                &mut transcript,
4309                                &mut compacted_this_turn,
4310                                &mut context_limit_recovery_attempts,
4311                            )
4312                            .await?
4313                        {
4314                            continue 'tool_rounds;
4315                        }
4316                        return Err(err);
4317                    }
4318                };
4319
4320                let inference_timestamp = OffsetDateTime::now_utc();
4321                self.emit(RoderEvent::InferenceEventReceived(InferenceEventReceived {
4322                    thread_id: req.thread_id.clone(),
4323                    turn_id: turn_id.clone(),
4324                    event: event.clone(),
4325                    timestamp: inference_timestamp,
4326                }))
4327                .await;
4328
4329                match event {
4330                    InferenceEvent::MessageDelta(delta) => {
4331                        if !delta.text.trim().is_empty() {
4332                            self.goals.record_activity(&turn_id).await;
4333                        }
4334                        if let Some((team_id, member)) =
4335                            self.teams.member_for_thread(&req.thread_id).await
4336                        {
4337                            self.emit(RoderEvent::TeamMemberMessageDelta(TeamMemberMessageDelta {
4338                                team_id,
4339                                member_id: member.id,
4340                                member_thread_id: req.thread_id.clone(),
4341                                turn_id: turn_id.clone(),
4342                                delta: delta.text.clone(),
4343                                timestamp: OffsetDateTime::now_utc(),
4344                            }))
4345                            .await;
4346                        }
4347                        if is_final_answer_phase(delta.phase.as_deref()) {
4348                            assistant_text.push_str(&delta.text);
4349                        } else if let Some(last) = phase_messages.last_mut()
4350                            && last.phase == delta.phase
4351                        {
4352                            last.text.push_str(&delta.text);
4353                        } else {
4354                            phase_messages.push(AssistantMessage {
4355                                text: delta.text,
4356                                phase: delta.phase,
4357                            });
4358                        }
4359                    }
4360                    InferenceEvent::ReasoningDelta(delta) => reasoning_text.push_str(&delta.text),
4361                    InferenceEvent::ToolCallCompleted(call) => {
4362                        self.goals.record_activity(&turn_id).await;
4363                        if completed_call_replayed(&transcript, &call)? {
4364                            replayed_tool_call = true;
4365                            continue;
4366                        }
4367                        if let Some(progress) = patch_progress.complete(
4368                            &req.thread_id,
4369                            &turn_id,
4370                            &call.id,
4371                            &call.name,
4372                            &call.arguments,
4373                        ) {
4374                            self.emit(RoderEvent::PatchProgress(progress)).await;
4375                        }
4376                        if !tool_calls
4377                            .iter()
4378                            .any(|previous: &ToolCallCompleted| previous.id == call.id)
4379                        {
4380                            eager_tools
4381                                .schedule(
4382                                    self,
4383                                    &output_context,
4384                                    &call,
4385                                    workspace.as_str(),
4386                                    turn_deadline,
4387                                    &mut transcript,
4388                                )
4389                                .await?;
4390                            tool_calls.push(call);
4391                        }
4392                    }
4393                    InferenceEvent::OutputItemCompleted(item) => {
4394                        sampling_output
4395                            .complete_item(self, &output_context, item, &mut transcript)
4396                            .await?;
4397                    }
4398                    InferenceEvent::Failed(failure) => {
4399                        speed_policy.record_failure();
4400                        let error = failure.message;
4401                        if self
4402                            .try_recover_from_context_limit(
4403                                &req.thread_id,
4404                                &turn_id,
4405                                &provider,
4406                                &model,
4407                                &error,
4408                                false,
4409                                &mut transcript,
4410                                &mut compacted_this_turn,
4411                                &mut context_limit_recovery_attempts,
4412                            )
4413                            .await?
4414                        {
4415                            continue 'tool_rounds;
4416                        }
4417                        self.persist_turn_item(
4418                            &req.thread_id,
4419                            &turn_id,
4420                            &TranscriptItem::Error(ErrorRecord {
4421                                message: error.clone(),
4422                            }),
4423                        )
4424                        .await?;
4425                        self.emit(RoderEvent::TurnFailed(TurnFailed {
4426                            thread_id: req.thread_id.clone(),
4427                            turn_id: turn_id.clone(),
4428                            error: error.clone(),
4429                            error_kind: None,
4430                            usage: None,
4431                            timestamp: OffsetDateTime::now_utc(),
4432                        }))
4433                        .await;
4434                        self.complete_team_member_turn_with_result(
4435                            &req.thread_id,
4436                            &turn_id,
4437                            TeamMemberStatus::Failed,
4438                            (!assistant_text.trim().is_empty()).then(|| assistant_text.clone()),
4439                            Some(error),
4440                        )
4441                        .await?;
4442                        return Ok(TurnRunOutcome::Stopped);
4443                    }
4444                    InferenceEvent::Usage(_) => {
4445                        self.sync_turn_goal_usage(&req.thread_id, &turn_id).await?;
4446                    }
4447                    InferenceEvent::Completed(metadata) => {
4448                        sampling_output
4449                            .finish(
4450                                self,
4451                                &output_context,
4452                                &assistant_text,
4453                                &phase_messages,
4454                                &reasoning_text,
4455                                &mut transcript,
4456                            )
4457                            .await?;
4458                        provider_stream_retry_attempts = 0;
4459                        turn_finish_reason = metadata
4460                            .stop_reason
4461                            .as_deref()
4462                            .map(finish_reason_from_stop_reason);
4463                        break;
4464                    }
4465                    InferenceEvent::Compaction(progress) => {
4466                        // Persist the boundary as soon as the provider emits a
4467                        // completed compaction item. ChatGPT Codex often aborts
4468                        // the SSE stream right after ("error decoding response
4469                        // body"), so waiting for ProviderMetadata loses the
4470                        // boundary and every subsequent request re-compacts.
4471                        if progress.status == "completed"
4472                            && let Some(item) = progress.item
4473                        {
4474                            let already = stream_compaction_item
4475                                .as_ref()
4476                                .and_then(|existing| {
4477                                    existing.get("id").and_then(serde_json::Value::as_str)
4478                                })
4479                                .zip(item.get("id").and_then(serde_json::Value::as_str))
4480                                .is_some_and(|(a, b)| a == b);
4481                            if !already {
4482                                stream_compaction_item = Some(item.clone());
4483                                let boundary = TranscriptItem::ProviderMetadata(
4484                                    crate::compaction::provider_metadata_from_compaction_item(item),
4485                                );
4486                                self.persist_turn_item(&req.thread_id, &turn_id, &boundary)
4487                                    .await?;
4488                                transcript.push(boundary);
4489                                transcript =
4490                                    crate::compaction::trim_to_last_compaction_boundary(transcript);
4491                                compacted_this_turn = true;
4492                            }
4493                        }
4494                    }
4495                    InferenceEvent::HostedToolCallStarted(_)
4496                    | InferenceEvent::HostedToolCallCompleted(_) => {}
4497                    InferenceEvent::ToolCallStarted(call) => {
4498                        patch_progress.start(&call.id, &call.name)
4499                    }
4500                    InferenceEvent::ToolCallDelta(delta) => {
4501                        if let Some(progress) = patch_progress.delta(
4502                            &req.thread_id,
4503                            &turn_id,
4504                            &delta.id,
4505                            &delta.arguments_delta,
4506                        ) {
4507                            self.emit(RoderEvent::PatchProgress(progress)).await;
4508                        }
4509                    }
4510                    InferenceEvent::ProviderMetadata(mut metadata) => {
4511                        if metadata.get("kind").and_then(serde_json::Value::as_str)
4512                            == Some("reliability_retry_attempt")
4513                        {
4514                            provider_stream_retry_attempts =
4515                                provider_stream_retry_attempts.saturating_add(1);
4516                        }
4517                        provider_requests_continuation |= metadata
4518                            .get("end_turn")
4519                            .and_then(serde_json::Value::as_bool)
4520                            == Some(false);
4521                        if let Some(compaction_item) = stream_compaction_item.as_ref() {
4522                            let _ = crate::compaction::ensure_provider_metadata_compaction(
4523                                &mut metadata,
4524                                compaction_item,
4525                            );
4526                        }
4527                        if metadata.get("output").is_none() {
4528                            let item = TranscriptItem::ProviderMetadata(metadata);
4529                            self.persist_turn_item(&req.thread_id, &turn_id, &item)
4530                                .await?;
4531                            transcript.push(item);
4532                        } else {
4533                            provider_metadata = Some(metadata);
4534                        }
4535                    }
4536                }
4537            }
4538
4539            speed_policy.record_model_output(
4540                !assistant_text.is_empty() || !phase_messages.is_empty(),
4541                tool_calls.len(),
4542            );
4543            if tool_calls.is_empty() {
4544                if provider_requests_continuation || replayed_tool_call {
4545                    if let Some(metadata) = provider_metadata {
4546                        let item = TranscriptItem::ProviderMetadata(metadata);
4547                        self.persist_turn_item(&req.thread_id, &turn_id, &item)
4548                            .await?;
4549                        transcript.push(item);
4550                    }
4551                    continue 'tool_rounds;
4552                }
4553                let steers = self.drain_turn_steers(&turn_id, &mut steering).await;
4554                if !steers.is_empty() {
4555                    if let Some(metadata) = provider_metadata {
4556                        let had_provider_compaction =
4557                            crate::compaction::provider_metadata_has_compaction(&metadata);
4558                        let item = TranscriptItem::ProviderMetadata(metadata);
4559                        self.persist_turn_item(&req.thread_id, &turn_id, &item)
4560                            .await?;
4561                        transcript.push(item);
4562                        if had_provider_compaction {
4563                            // Server-side compaction replaced the prior window;
4564                            // drop pre-boundary items so the next request does
4565                            // not re-send (and re-compact) the full history.
4566                            transcript =
4567                                crate::compaction::trim_to_last_compaction_boundary(transcript);
4568                            compacted_this_turn = true;
4569                        }
4570                    }
4571                    self.append_steers(&req, &turn_id, &mut transcript, steers)
4572                        .await?;
4573                    continue;
4574                }
4575                if !deadline_finalization_requested
4576                    && req.task_ledger_required
4577                    && runtime_profile == RuntimeProfile::Eval
4578                    && task_ledger_completion_reminders < TASK_LEDGER_COMPLETION_REMINDER_LIMIT
4579                    && (!assistant_text.trim().is_empty() || !phase_messages.is_empty())
4580                    && let Some(prompt) = task_ledger_completion_prompt(&transcript)
4581                {
4582                    task_ledger_completion_reminders += 1;
4583                    let item = TranscriptItem::UserMessage(UserMessage::text(prompt));
4584                    self.persist_turn_item(&req.thread_id, &turn_id, &item)
4585                        .await?;
4586                    transcript.push(item);
4587                    continue;
4588                }
4589                if !deadline_finalization_requested
4590                    && let Some(prompt) = verification_gate.blocking_prompt()
4591                {
4592                    speed_policy.record_verification_required();
4593                    self.emit(RoderEvent::VerificationRequired(VerificationRequired {
4594                        thread_id: req.thread_id.clone(),
4595                        turn_id: turn_id.clone(),
4596                        reason: verification_gate.reason(),
4597                        changed_files: verification_gate.changed_files(),
4598                        tool_evidence: verification_gate.tool_evidence.clone(),
4599                        tests_run: verification_gate.tests_run.clone(),
4600                        open_gaps: verification_gate.open_gaps.clone(),
4601                        timestamp: OffsetDateTime::now_utc(),
4602                    }))
4603                    .await;
4604                    let item = TranscriptItem::UserMessage(UserMessage::text(prompt));
4605                    self.persist_turn_item(&req.thread_id, &turn_id, &item)
4606                        .await?;
4607                    transcript.push(item);
4608                    continue;
4609                }
4610                // Loop persistence nudge: in non-interactive/eval runs, when the
4611                // model returns a final message with no tool calls, gently prompt
4612                // it to keep going (bounded by `empty_tool_call_nudges`) before
4613                // ending the turn. Gated to non-interactive/eval so interactive
4614                // chat completions are never delayed, and only fires when the
4615                // model actually produced a final answer to re-examine.
4616                if !deadline_finalization_requested
4617                    && runtime_profile != RuntimeProfile::Interactive
4618                    && cfg.reliability.empty_tool_call_nudges > 0
4619                    && empty_tool_call_nudges_used < cfg.reliability.empty_tool_call_nudges
4620                    && (!assistant_text.trim().is_empty() || !phase_messages.is_empty())
4621                {
4622                    empty_tool_call_nudges_used += 1;
4623                    let item = TranscriptItem::UserMessage(UserMessage::text(
4624                        RELIABILITY_CONTINUATION_PROMPT.to_string(),
4625                    ));
4626                    self.persist_turn_item(&req.thread_id, &turn_id, &item)
4627                        .await?;
4628                    transcript.push(item);
4629                    continue;
4630                }
4631                if deadline_finalization_requested
4632                    && assistant_text.trim().is_empty()
4633                    && phase_messages.is_empty()
4634                {
4635                    assistant_text = format!(
4636                        "Deadline finalization completed without model text. {}",
4637                        turn_partial_result(&transcript)
4638                    );
4639                }
4640                final_assistant_text = assistant_text;
4641                final_provider_metadata = provider_metadata;
4642                exhausted_tool_rounds = false;
4643                break;
4644            }
4645
4646            if let Some(metadata) = provider_metadata {
4647                let had_provider_compaction =
4648                    crate::compaction::provider_metadata_has_compaction(&metadata);
4649                let item = TranscriptItem::ProviderMetadata(metadata);
4650                self.persist_turn_item(&req.thread_id, &turn_id, &item)
4651                    .await?;
4652                transcript.push(item);
4653                if had_provider_compaction {
4654                    // Server-side compaction replaced the prior window; drop
4655                    // pre-boundary items so subsequent tool rounds use the
4656                    // compact window as the next input.
4657                    transcript = crate::compaction::trim_to_last_compaction_boundary(transcript);
4658                    compacted_this_turn = true;
4659                }
4660            }
4661            eager_tools.drain().await?;
4662            for call in &tool_calls {
4663                if eager_tools.scheduled(&call.id) {
4664                    continue;
4665                }
4666                let tool_item = TranscriptItem::ToolCall(ToolCallRecord {
4667                    id: call.id.clone(),
4668                    name: call.name.clone(),
4669                    arguments: call.arguments.clone(),
4670                });
4671                self.persist_turn_item(&req.thread_id, &turn_id, &tool_item)
4672                    .await?;
4673                transcript.push(tool_item);
4674                self.persist_model_profile_segment(
4675                    &req.thread_id,
4676                    &turn_id,
4677                    model_profile.as_ref(),
4678                    &provider,
4679                    &model,
4680                    "tool_call",
4681                )
4682                .await?;
4683            }
4684            if let Some(deadline) = turn_deadline
4685                && deadline_expired(deadline)
4686            {
4687                self.fail_turn_due_to_deadline(&req.thread_id, &turn_id, deadline, &transcript)
4688                    .await?;
4689                return Ok(TurnRunOutcome::Stopped);
4690            }
4691            let mut results = eager_tools.take_results(&tool_calls);
4692            let remaining_calls = tool_calls
4693                .into_iter()
4694                .filter(|call| !eager_tools.scheduled(&call.id))
4695                .collect();
4696            results.extend(
4697                self.route_tool_calls(
4698                    &req.thread_id,
4699                    &turn_id,
4700                    remaining_calls,
4701                    parallel_tool_calls,
4702                    Some(workspace.as_str()),
4703                    turn_deadline,
4704                )
4705                .await?,
4706            );
4707            let reliability_limit = reliability.record_tool_results(
4708                &cfg.reliability,
4709                &results,
4710                runtime_profile == RuntimeProfile::Interactive,
4711            );
4712            for result in results {
4713                verification_gate.record_tool_result(&result);
4714                transcript.push(TranscriptItem::ToolResult(result));
4715                self.persist_model_profile_segment(
4716                    &req.thread_id,
4717                    &turn_id,
4718                    model_profile.as_ref(),
4719                    &provider,
4720                    &model,
4721                    "tool_result",
4722                )
4723                .await?;
4724            }
4725            if let Some(limit) = reliability_limit {
4726                if limit.decision == ReliabilityLimitDecision::RequestContinuation {
4727                    // Loop persistence: rather than ending the turn, reset the
4728                    // consecutive-failure counter, nudge the model to keep going,
4729                    // and continue the round loop. Bounded by the per-turn tool
4730                    // failure ceiling, `max_model_calls_per_turn`, and
4731                    // `MAX_TOOL_ROUNDS_PER_TURN`.
4732                    self.record_reliability_limit_continuation(
4733                        &req.thread_id,
4734                        &turn_id,
4735                        &provider,
4736                        &model,
4737                        limit,
4738                    )
4739                    .await;
4740                    reliability.reset_consecutive_failures();
4741                    let nudge = TranscriptItem::UserMessage(UserMessage::text(
4742                        RELIABILITY_CONTINUATION_PROMPT.to_string(),
4743                    ));
4744                    self.persist_turn_item(&req.thread_id, &turn_id, &nudge)
4745                        .await?;
4746                    transcript.push(nudge);
4747                } else {
4748                    self.fail_turn_due_to_reliability_limit(
4749                        &req.thread_id,
4750                        &turn_id,
4751                        &provider,
4752                        &model,
4753                        limit,
4754                        &transcript,
4755                    )
4756                    .await?;
4757                    return Ok(TurnRunOutcome::Stopped);
4758                }
4759            }
4760            let compaction_generation_before = self.compaction_generation(&req.thread_id);
4761            transcript = self
4762                .compact_transcript_if_needed(
4763                    &req.thread_id,
4764                    &turn_id,
4765                    &provider,
4766                    &model,
4767                    transcript,
4768                    self.compaction_options_for_turn(&req.thread_id, !compacted_this_turn),
4769                )
4770                .await?;
4771            compacted_this_turn |=
4772                self.compaction_generation(&req.thread_id) != compaction_generation_before;
4773        }
4774
4775        if exhausted_tool_rounds {
4776            let message =
4777                format!("tool call limit reached after {MAX_TOOL_ROUNDS_PER_TURN} rounds");
4778            self.persist_turn_item(
4779                &req.thread_id,
4780                &turn_id,
4781                &TranscriptItem::Error(ErrorRecord {
4782                    message: message.clone(),
4783                }),
4784            )
4785            .await?;
4786            self.emit(RoderEvent::TurnFailed(TurnFailed {
4787                thread_id: req.thread_id.clone(),
4788                turn_id: turn_id.clone(),
4789                error: message.clone(),
4790                error_kind: None,
4791                usage: None,
4792                timestamp: OffsetDateTime::now_utc(),
4793            }))
4794            .await;
4795            self.complete_team_member_turn_with_result(
4796                &req.thread_id,
4797                &turn_id,
4798                TeamMemberStatus::Failed,
4799                None,
4800                Some(message),
4801            )
4802            .await?;
4803            return Ok(TurnRunOutcome::Stopped);
4804        }
4805
4806        if let Some(metadata) = final_provider_metadata {
4807            self.persist_turn_item(
4808                &req.thread_id,
4809                &turn_id,
4810                &TranscriptItem::ProviderMetadata(metadata),
4811            )
4812            .await?;
4813        }
4814
4815        let (cleanup, ownership) = self.await_provider_turn_cleanup(&turn_id).await;
4816        self.record_turn_lifecycle_with_ownership(
4817            req.thread_id.clone(),
4818            turn_id.clone(),
4819            TurnLifecycleState::Completed,
4820            cleanup,
4821            None,
4822            ownership,
4823        )
4824        .await;
4825        self.emit(RoderEvent::TurnCompleted(TurnCompleted {
4826            thread_id: req.thread_id.clone(),
4827            turn_id: turn_id.clone(),
4828            usage: None,
4829            finish_reason: turn_finish_reason,
4830            timestamp: OffsetDateTime::now_utc(),
4831        }))
4832        .await;
4833        self.complete_team_member_turn_with_result(
4834            &req.thread_id,
4835            &turn_id,
4836            TeamMemberStatus::Completed,
4837            (!final_assistant_text.is_empty()).then_some(final_assistant_text),
4838            None,
4839        )
4840        .await?;
4841        Ok(TurnRunOutcome::Completed)
4842    }
4843
4844    async fn route_tool_calls(
4845        self: &Arc<Self>,
4846        thread_id: &ThreadId,
4847        turn_id: &TurnId,
4848        calls: Vec<ToolCallCompleted>,
4849        parallel: bool,
4850        workspace: Option<&str>,
4851        deadline: Option<OffsetDateTime>,
4852    ) -> anyhow::Result<Vec<ToolResultRecord>> {
4853        // Enforce the agent_swarm exclusivity rule (roadmap 104, Task 2):
4854        // `agent_swarm` must be the only tool call in a model response. A mixed
4855        // or multi-swarm batch is denied wholesale with actionable retry text so
4856        // the model re-issues `agent_swarm` by itself, and every tool_call_id
4857        // still gets a response (keeping the chat-completions transcript valid).
4858        if let Some(violation) = roder_api::subagents::agent_swarm_batch_violation(
4859            calls.iter().map(|call| call.name.as_str()),
4860        ) {
4861            let message = violation.deny_message();
4862            return Ok(calls
4863                .into_iter()
4864                .map(|call| ToolResultRecord {
4865                    id: call.id,
4866                    name: Some(call.name),
4867                    result: message.clone(),
4868                    display_payload: None,
4869                    is_error: true,
4870                })
4871                .collect());
4872        }
4873        // Emit swarm lifecycle events on the bus (roadmap 104, Task 1) so any
4874        // app-server/SDK/TUI client can observe a swarm as a whole; per-child
4875        // progress flows through the existing Subagent* trace events.
4876        let swarm_call = (calls.len() == 1
4877            && calls[0].name == roder_api::subagents::AGENT_SWARM_TOOL_NAME)
4878            .then(|| (calls[0].id.clone(), calls[0].arguments.clone()));
4879        if let Some((tool_id, args)) = &swarm_call {
4880            self.emit(RoderEvent::AgentSwarmStarted(
4881                roder_api::subagents::AgentSwarmStarted {
4882                    thread_id: thread_id.clone(),
4883                    turn_id: turn_id.clone(),
4884                    tool_id: tool_id.clone(),
4885                    child_count: agent_swarm_child_count(args),
4886                    timestamp: OffsetDateTime::now_utc(),
4887                },
4888            ))
4889            .await;
4890        }
4891
4892        let force_sequential = calls
4893            .iter()
4894            .any(|call| crate::agent_control_tools::is_agent_control_tool(&call.name));
4895        let results = if parallel && !force_sequential {
4896            try_join_all(calls.into_iter().map(|call| {
4897                self.route_tool_call(thread_id, turn_id, call, workspace, deadline, None)
4898            }))
4899            .await
4900        } else {
4901            let mut results = Vec::with_capacity(calls.len());
4902            for call in calls {
4903                results.push(
4904                    self.route_tool_call(thread_id, turn_id, call, workspace, deadline, None)
4905                        .await?,
4906                );
4907            }
4908            Ok(results)
4909        }?;
4910
4911        if let Some((tool_id, _)) = &swarm_call
4912            && let Some(result) = results.iter().find(|result| &result.id == tool_id)
4913            && let Some((completed, failed, aborted)) = parse_swarm_counts(&result.result)
4914        {
4915            self.emit(RoderEvent::AgentSwarmCompleted(
4916                roder_api::subagents::AgentSwarmCompleted {
4917                    thread_id: thread_id.clone(),
4918                    turn_id: turn_id.clone(),
4919                    tool_id: tool_id.clone(),
4920                    completed,
4921                    failed,
4922                    aborted,
4923                    timestamp: OffsetDateTime::now_utc(),
4924                },
4925            ))
4926            .await;
4927        }
4928
4929        Ok(results)
4930    }
4931
4932    /// On provider context/prompt-length failures, force-compact (and if needed
4933    /// strip the last bulky item) then retry the current tool round instead of
4934    /// failing the turn. Bounded so a permanently oversized suffix still stops.
4935    async fn try_recover_from_context_limit(
4936        &self,
4937        thread_id: &ThreadId,
4938        turn_id: &TurnId,
4939        provider: &str,
4940        model: &str,
4941        error: &str,
4942        typed_overflow: bool,
4943        transcript: &mut Vec<TranscriptItem>,
4944        compacted_this_turn: &mut bool,
4945        recovery_attempts: &mut u32,
4946    ) -> anyhow::Result<bool> {
4947        const MAX_CONTEXT_LIMIT_RECOVERY_ATTEMPTS: u32 = 2;
4948        if !typed_overflow && !crate::compaction::is_context_limit_failure_message(error) {
4949            return Ok(false);
4950        }
4951        if *recovery_attempts >= MAX_CONTEXT_LIMIT_RECOVERY_ATTEMPTS {
4952            return Ok(false);
4953        }
4954        *recovery_attempts = recovery_attempts.saturating_add(1);
4955
4956        let error_item = TranscriptItem::Error(ErrorRecord {
4957            message: error.to_string(),
4958        });
4959        self.persist_turn_item(thread_id, turn_id, &error_item)
4960            .await?;
4961        transcript.push(error_item);
4962
4963        // After the first failed recovery, drop the item ahead of the latest
4964        // user/error so a second compact can succeed when the suffix itself is
4965        // still oversized (e.g. a huge tool result right before the prompt).
4966        if *recovery_attempts > 1 {
4967            *transcript =
4968                crate::compaction::strip_last_message_before_error(std::mem::take(transcript));
4969        }
4970
4971        let force_options = crate::compaction::CompactionOptions {
4972            allow_repeat: true,
4973            force: true,
4974            hysteresis_baseline: None,
4975            preserve_hint: Some(
4976                "Recover from a provider context/prompt-length limit; preserve the active user goal and recent tool outcomes."
4977                    .to_string(),
4978            ),
4979        };
4980        let before_len = transcript.len();
4981        let tool_images = self.tool_images(provider, model);
4982        let before_tokens = crate::compaction::estimate_prompt_tokens(transcript, tool_images);
4983        *transcript = self
4984            .compact_transcript_if_needed(
4985                thread_id,
4986                turn_id,
4987                provider,
4988                model,
4989                std::mem::take(transcript),
4990                force_options,
4991            )
4992            .await?;
4993        let after_tokens = crate::compaction::estimate_prompt_tokens(transcript, tool_images);
4994        *compacted_this_turn = *compacted_this_turn
4995            || transcript
4996                .iter()
4997                .any(crate::compaction::is_compaction_boundary);
4998
4999        self.emit(RoderEvent::ReliabilityRetryRecorded(
5000            ReliabilityRetryRecorded {
5001                context: ReliabilityContext {
5002                    thread_id: thread_id.clone(),
5003                    turn_id: turn_id.clone(),
5004                    provider: Some(provider.to_string()),
5005                    model: Some(model.to_string()),
5006                    ..ReliabilityContext::default()
5007                },
5008                error_class: ReliabilityErrorClass::ProviderError,
5009                decision: ReliabilityRetryDecision::Retry,
5010                attempt: *recovery_attempts,
5011                max_attempts: MAX_CONTEXT_LIMIT_RECOVERY_ATTEMPTS,
5012                delay_ms: Some(0),
5013                details: ReliabilityDetails::redacted(format!(
5014                    "context_limit_recovery: compact {before_tokens}->{after_tokens} tokens (items {before_len}->{}); {error}",
5015                    transcript.len()
5016                )),
5017                timestamp: OffsetDateTime::now_utc(),
5018            },
5019        ))
5020        .await;
5021
5022        // If force-compact did not shrink anything and we still have budget,
5023        // strip once more so the next round is not identical.
5024        if after_tokens >= before_tokens && *recovery_attempts < MAX_CONTEXT_LIMIT_RECOVERY_ATTEMPTS
5025        {
5026            *transcript =
5027                crate::compaction::strip_last_message_before_error(std::mem::take(transcript));
5028        }
5029        Ok(true)
5030    }
5031
5032    async fn fail_turn_with_error(
5033        &self,
5034        thread_id: &ThreadId,
5035        turn_id: &TurnId,
5036        message: String,
5037    ) -> anyhow::Result<()> {
5038        self.persist_turn_item(
5039            thread_id,
5040            turn_id,
5041            &TranscriptItem::Error(ErrorRecord {
5042                message: message.clone(),
5043            }),
5044        )
5045        .await?;
5046        self.emit(RoderEvent::TurnFailed(TurnFailed {
5047            thread_id: thread_id.clone(),
5048            turn_id: turn_id.clone(),
5049            error: message.clone(),
5050            error_kind: None,
5051            usage: None,
5052            timestamp: OffsetDateTime::now_utc(),
5053        }))
5054        .await;
5055        self.complete_team_member_turn_with_result(
5056            thread_id,
5057            turn_id,
5058            TeamMemberStatus::Failed,
5059            None,
5060            Some(message),
5061        )
5062        .await?;
5063        Ok(())
5064    }
5065
5066    async fn fail_turn_due_to_deadline(
5067        &self,
5068        thread_id: &ThreadId,
5069        turn_id: &TurnId,
5070        deadline: OffsetDateTime,
5071        transcript: &[TranscriptItem],
5072    ) -> anyhow::Result<()> {
5073        let partial_result = turn_partial_result(transcript);
5074        self.emit(RoderEvent::TurnPartialResult(TurnPartialResult {
5075            thread_id: thread_id.clone(),
5076            turn_id: turn_id.clone(),
5077            summary: partial_result.clone(),
5078            timestamp: OffsetDateTime::now_utc(),
5079        }))
5080        .await;
5081        self.emit(RoderEvent::TurnDeadlineExceeded(TurnDeadlineExceeded {
5082            thread_id: thread_id.clone(),
5083            turn_id: turn_id.clone(),
5084            deadline,
5085            partial_result: partial_result.clone(),
5086            timestamp: OffsetDateTime::now_utc(),
5087        }))
5088        .await;
5089        let message = "turn deadline expired".to_string();
5090        self.persist_turn_item(
5091            thread_id,
5092            turn_id,
5093            &TranscriptItem::Error(ErrorRecord {
5094                message: format!("{message}: {partial_result}"),
5095            }),
5096        )
5097        .await?;
5098        self.emit(RoderEvent::TurnFailed(TurnFailed {
5099            thread_id: thread_id.clone(),
5100            turn_id: turn_id.clone(),
5101            error: message.clone(),
5102            error_kind: Some("deadline_timeout".to_string()),
5103            usage: None,
5104            timestamp: OffsetDateTime::now_utc(),
5105        }))
5106        .await;
5107        self.complete_team_member_turn_with_result(
5108            thread_id,
5109            turn_id,
5110            TeamMemberStatus::Failed,
5111            None,
5112            Some(format!("{message}: {partial_result}")),
5113        )
5114        .await?;
5115        Ok(())
5116    }
5117
5118    async fn start_deadline_finalization(
5119        &self,
5120        thread_id: &ThreadId,
5121        turn_id: &TurnId,
5122        transcript: &mut Vec<TranscriptItem>,
5123        remaining_seconds: u64,
5124    ) -> anyhow::Result<()> {
5125        let item = TranscriptItem::UserMessage(crate::deadline_policy::finalization_message(
5126            remaining_seconds,
5127        ));
5128        self.persist_turn_item(thread_id, turn_id, &item).await?;
5129        transcript.push(item);
5130        self.emit(RoderEvent::TurnPartialResult(TurnPartialResult {
5131            thread_id: thread_id.clone(),
5132            turn_id: turn_id.clone(),
5133            summary: turn_partial_result(transcript),
5134            timestamp: OffsetDateTime::now_utc(),
5135        }))
5136        .await;
5137        Ok(())
5138    }
5139
5140    /// Emits the reliability-limit event for a continuation (loop persistence)
5141    /// without failing the turn. The caller resets the failure counter and
5142    /// injects a nudge so the round loop keeps going.
5143    async fn record_reliability_limit_continuation(
5144        &self,
5145        thread_id: &ThreadId,
5146        turn_id: &TurnId,
5147        provider: &str,
5148        model: &str,
5149        limit: ReliabilityLimitHit,
5150    ) {
5151        self.emit(RoderEvent::ReliabilityLimitRecorded(
5152            ReliabilityLimitRecorded {
5153                context: ReliabilityContext {
5154                    thread_id: thread_id.clone(),
5155                    turn_id: turn_id.clone(),
5156                    tool_id: None,
5157                    tool_name: None,
5158                    provider: Some(provider.to_string()),
5159                    model: Some(model.to_string()),
5160                },
5161                error_class: limit.error_class,
5162                limit_kind: limit.limit_kind,
5163                decision: limit.decision,
5164                current: limit.current,
5165                limit: limit.limit,
5166                details: ReliabilityDetails::redacted(&limit.message),
5167                timestamp: OffsetDateTime::now_utc(),
5168            },
5169        ))
5170        .await;
5171    }
5172
5173    async fn fail_turn_due_to_reliability_limit(
5174        &self,
5175        thread_id: &ThreadId,
5176        turn_id: &TurnId,
5177        provider: &str,
5178        model: &str,
5179        limit: ReliabilityLimitHit,
5180        transcript: &[TranscriptItem],
5181    ) -> anyhow::Result<()> {
5182        self.emit(RoderEvent::ReliabilityLimitRecorded(
5183            ReliabilityLimitRecorded {
5184                context: ReliabilityContext {
5185                    thread_id: thread_id.clone(),
5186                    turn_id: turn_id.clone(),
5187                    tool_id: None,
5188                    tool_name: None,
5189                    provider: Some(provider.to_string()),
5190                    model: Some(model.to_string()),
5191                },
5192                error_class: limit.error_class,
5193                limit_kind: limit.limit_kind,
5194                decision: limit.decision,
5195                current: limit.current,
5196                limit: limit.limit,
5197                details: ReliabilityDetails::redacted(&limit.message),
5198                timestamp: OffsetDateTime::now_utc(),
5199            },
5200        ))
5201        .await;
5202        let partial_result = turn_partial_result(transcript);
5203        self.emit(RoderEvent::TurnPartialResult(TurnPartialResult {
5204            thread_id: thread_id.clone(),
5205            turn_id: turn_id.clone(),
5206            summary: partial_result.clone(),
5207            timestamp: OffsetDateTime::now_utc(),
5208        }))
5209        .await;
5210        let message = format!("reliability limit reached: {}", limit.message);
5211        self.persist_turn_item(
5212            thread_id,
5213            turn_id,
5214            &TranscriptItem::Error(ErrorRecord {
5215                message: format!("{message}: {partial_result}"),
5216            }),
5217        )
5218        .await?;
5219        self.emit(RoderEvent::TurnFailed(TurnFailed {
5220            thread_id: thread_id.clone(),
5221            turn_id: turn_id.clone(),
5222            error: message.clone(),
5223            error_kind: Some("reliability_limit".to_string()),
5224            usage: None,
5225            timestamp: OffsetDateTime::now_utc(),
5226        }))
5227        .await;
5228        self.complete_team_member_turn_with_result(
5229            thread_id,
5230            turn_id,
5231            TeamMemberStatus::Failed,
5232            None,
5233            Some(format!("{message}: {partial_result}")),
5234        )
5235        .await?;
5236        Ok(())
5237    }
5238
5239    async fn append_steers(
5240        &self,
5241        req: &StartTurnRequest,
5242        turn_id: &TurnId,
5243        transcript: &mut Vec<TranscriptItem>,
5244        steers: Vec<QueuedTurnSteer>,
5245    ) -> anyhow::Result<()> {
5246        for queued in steers {
5247            let mut steer = queued.message;
5248            steer.text = steer.text.trim().to_string();
5249            if steer.text.is_empty() && steer.images.is_empty() {
5250                continue;
5251            }
5252            let item = TranscriptItem::UserMessage(steer);
5253            self.persist_turn_item(&req.thread_id, turn_id, &item)
5254                .await?;
5255            transcript.push(item);
5256            if let Some(ack) = queued.mailbox_ack {
5257                self.teams
5258                    .mark_mailbox_messages_delivered(&ack.team_id, turn_id, &ack.message_ids)
5259                    .await?;
5260            }
5261        }
5262        Ok(())
5263    }
5264
5265    async fn persist_model_profile_segment(
5266        &self,
5267        thread_id: &ThreadId,
5268        turn_id: &TurnId,
5269        profile: Option<&ModelHarnessProfile>,
5270        provider: &str,
5271        model: &str,
5272        segment: &str,
5273    ) -> anyhow::Result<()> {
5274        let item = TranscriptItem::ProviderMetadata(model_profile_segment_metadata(
5275            profile, provider, model, segment,
5276        ));
5277        self.persist_turn_item(thread_id, turn_id, &item).await
5278    }
5279
5280    fn task_ledger_tool_specs(
5281        &self,
5282        profile: Option<&ModelHarnessProfile>,
5283    ) -> Vec<roder_api::tools::ToolSpec> {
5284        self.tool_registry
5285            .get(TASK_LEDGER_TOOL_NAME)
5286            .map(|tool| {
5287                tool.spec()
5288                    .normalized_for_model_profile(schema_policy_for_model(profile))
5289            })
5290            .into_iter()
5291            .collect()
5292    }
5293
5294    pub(crate) fn engine_for(&self, provider: &str) -> anyhow::Result<Arc<dyn InferenceEngine>> {
5295        self.registry
5296            .inference_engine(provider)
5297            .or_else(|| {
5298                self.registry
5299                    .default_inference_engine()
5300                    .filter(|engine| provider.is_empty() || engine.id() == provider)
5301            })
5302            .ok_or_else(|| anyhow::anyhow!("inference provider {provider:?} is not registered"))
5303    }
5304
5305    pub async fn emit(&self, mut event: RoderEvent) -> EventEnvelope {
5306        self.account_usage_event(&mut event).await;
5307        self.dispatch_local_hook_lifecycle(&event).await;
5308        if let Some(record) = Self::lifecycle_record_for_event(&event) {
5309            let _ = self.persist_turn_lifecycle_record(&record).await;
5310        }
5311        let envelope = self.bus.emit(event);
5312        if let (Some(store), Some(thread_id)) = (&self.thread_store, envelope.thread_id.as_ref())
5313            && should_persist_thread_event(thread_id)
5314        {
5315            let _ = store.append_event(thread_id, &envelope).await;
5316        }
5317        self.dispatch_event_sinks(&envelope).await;
5318        envelope
5319    }
5320
5321    async fn dispatch_local_hook_lifecycle(&self, event: &RoderEvent) {
5322        // SessionStart belongs to the session, not the turn: firing it from
5323        // TurnStarted re-ran setup hooks on every user message. ThreadCreated
5324        // and ThreadLoaded carry no turn id, so they borrow a synthetic one the
5325        // way the SessionEnd path does.
5326        let session_turn_id = SESSION_HOOK_TURN_ID.to_string();
5327        let (thread_id, turn_id, name, matcher, input) = match event {
5328            RoderEvent::ThreadCreated(event) => (
5329                &event.thread_id,
5330                &session_turn_id,
5331                "SessionStart",
5332                Some("startup"),
5333                serde_json::json!({"source":"startup"}),
5334            ),
5335            RoderEvent::ThreadLoaded(event) => (
5336                &event.thread_id,
5337                &session_turn_id,
5338                "SessionStart",
5339                Some("resume"),
5340                serde_json::json!({"source":"resume"}),
5341            ),
5342            RoderEvent::TurnCompleted(event) => (
5343                &event.thread_id,
5344                &event.turn_id,
5345                "Stop",
5346                None,
5347                serde_json::json!({"finishReason":event.finish_reason}),
5348            ),
5349            RoderEvent::TurnFailed(event) => (
5350                &event.thread_id,
5351                &event.turn_id,
5352                "Stop",
5353                None,
5354                serde_json::json!({"error":event.error}),
5355            ),
5356            RoderEvent::ContextCompactionStarted(event) => (
5357                &event.thread_id,
5358                &event.turn_id,
5359                "PreCompact",
5360                None,
5361                serde_json::json!({"originalItemCount":event.original_item_count,"originalEstimatedTokens":event.original_estimated_tokens}),
5362            ),
5363            RoderEvent::ContextCompactionRecorded(event) => (
5364                &event.thread_id,
5365                &event.turn_id,
5366                "PostCompact",
5367                None,
5368                serde_json::json!({"compactedItemCount":event.compacted_item_count,"compactedEstimatedTokens":event.compacted_estimated_tokens}),
5369            ),
5370            RoderEvent::SubagentStarted(event) => (
5371                &event.parent_thread_id,
5372                &event.parent_turn_id,
5373                "SubagentStart",
5374                Some(event.agent_type.as_str()),
5375                serde_json::json!({"subagentThreadId":event.thread_id,"agentType":event.agent_type,"description":event.description}),
5376            ),
5377            RoderEvent::SubagentCompleted(event) => (
5378                &event.parent_thread_id,
5379                &event.parent_turn_id,
5380                "SubagentStop",
5381                Some(event.agent_type.as_str()),
5382                serde_json::json!({"subagentThreadId":event.thread_id,"agentType":event.agent_type,"exitReason":event.exit_reason}),
5383            ),
5384            RoderEvent::SubagentFailed(event) => (
5385                &event.parent_thread_id,
5386                &event.parent_turn_id,
5387                "SubagentStop",
5388                Some(event.agent_type.as_str()),
5389                serde_json::json!({"subagentThreadId":event.thread_id,"agentType":event.agent_type,"error":event.error}),
5390            ),
5391            _ => return,
5392        };
5393        // One SessionStart per session, whichever event opened it.
5394        if name == "SessionStart" && !self.claim_session_hook_start(thread_id).await {
5395            return;
5396        }
5397        let workspace = self.config.read().await.workspace.clone();
5398        crate::hooks::run_lifecycle(
5399            self,
5400            thread_id,
5401            turn_id,
5402            workspace.as_deref(),
5403            name,
5404            matcher,
5405            input,
5406        )
5407        .await;
5408    }
5409
5410    /// Claim the single `SessionStart` dispatch for a thread, returning whether
5411    /// this caller won it.
5412    pub(crate) async fn claim_session_hook_start(&self, thread_id: &ThreadId) -> bool {
5413        self.session_hook_started
5414            .write()
5415            .await
5416            .insert(thread_id.clone())
5417    }
5418
5419    /// Records a projected thread item event and persists it through the configured thread store.
5420    ///
5421    /// App-server protocol notification bridges currently call this after translating runtime
5422    /// events into item events. Headless runtime consumers that subscribe to `RoderEvent` directly
5423    /// must make the same call if they need the derived item event stream persisted.
5424    pub async fn record_thread_item_event_kind(
5425        &self,
5426        thread_id: &ThreadId,
5427        turn_id: &TurnId,
5428        timestamp: OffsetDateTime,
5429        kind: ThreadItemEventKind,
5430    ) -> anyhow::Result<ThreadItemEvent> {
5431        let seq = self.next_thread_item_event_seq(thread_id).await?;
5432        let item_event = ThreadItemEvent {
5433            seq,
5434            event_id: uuid::Uuid::new_v4().to_string(),
5435            thread_id: thread_id.clone(),
5436            turn_id: turn_id.clone(),
5437            timestamp,
5438            event: kind,
5439        };
5440        if let Some(store) = &self.thread_store {
5441            store.append_item_event(thread_id, &item_event).await?;
5442        }
5443        self.remember_thread_item_event(&item_event).await?;
5444        Ok(item_event)
5445    }
5446
5447    async fn next_thread_item_event_seq(&self, thread_id: &ThreadId) -> anyhow::Result<u64> {
5448        self.ensure_thread_item_cache(thread_id).await?;
5449        Ok(self
5450            .thread_item_cache
5451            .lock()
5452            .await
5453            .next_item_event_seq(thread_id))
5454    }
5455
5456    pub async fn thread_item_exists(
5457        &self,
5458        thread_id: &ThreadId,
5459        turn_id: &TurnId,
5460        item_id: &str,
5461    ) -> anyhow::Result<bool> {
5462        self.ensure_thread_item_cache(thread_id).await?;
5463        Ok(self
5464            .thread_item_cache
5465            .lock()
5466            .await
5467            .thread_item_exists(thread_id, turn_id, item_id))
5468    }
5469
5470    pub async fn current_reasoning_item_id(
5471        &self,
5472        thread_id: &ThreadId,
5473        turn_id: &TurnId,
5474    ) -> anyhow::Result<Option<String>> {
5475        self.ensure_thread_item_cache(thread_id).await?;
5476        Ok(self
5477            .thread_item_cache
5478            .lock()
5479            .await
5480            .current_reasoning_item_id(thread_id, turn_id))
5481    }
5482
5483    async fn remember_thread_item_event(&self, item_event: &ThreadItemEvent) -> anyhow::Result<()> {
5484        self.ensure_thread_item_cache(&item_event.thread_id).await?;
5485        self.thread_item_cache
5486            .lock()
5487            .await
5488            .remember_item_event(item_event);
5489        Ok(())
5490    }
5491
5492    pub async fn latest_transcript_item_index(
5493        &self,
5494        thread_id: &ThreadId,
5495        turn_id: &TurnId,
5496    ) -> anyhow::Result<Option<usize>> {
5497        self.ensure_thread_item_cache(thread_id).await?;
5498        Ok(self
5499            .thread_item_cache
5500            .lock()
5501            .await
5502            .latest_transcript_item_index(thread_id, turn_id))
5503    }
5504
5505    async fn next_transcript_item_index(
5506        &self,
5507        thread_id: &ThreadId,
5508        turn_id: &TurnId,
5509    ) -> anyhow::Result<usize> {
5510        self.ensure_thread_item_cache(thread_id).await?;
5511        Ok(self
5512            .thread_item_cache
5513            .lock()
5514            .await
5515            .next_transcript_item_index(thread_id, turn_id))
5516    }
5517
5518    async fn remember_transcript_item_index(
5519        &self,
5520        thread_id: &ThreadId,
5521        turn_id: &TurnId,
5522        item_index: usize,
5523    ) -> anyhow::Result<()> {
5524        self.ensure_thread_item_cache(thread_id).await?;
5525        self.thread_item_cache
5526            .lock()
5527            .await
5528            .remember_transcript_item_index(thread_id, turn_id, item_index);
5529        Ok(())
5530    }
5531
5532    async fn ensure_thread_item_cache(&self, thread_id: &ThreadId) -> anyhow::Result<()> {
5533        if self
5534            .thread_item_cache
5535            .lock()
5536            .await
5537            .contains_thread(thread_id)
5538        {
5539            return Ok(());
5540        }
5541
5542        let snapshot = if let Some(store) = &self.thread_store {
5543            store.load_thread(thread_id).await?
5544        } else {
5545            None
5546        };
5547        self.thread_item_cache.lock().await.ensure_thread(
5548            thread_id,
5549            ThreadItemCacheEntry::from_snapshot(snapshot.as_ref()),
5550        );
5551        Ok(())
5552    }
5553
5554    pub(crate) async fn persist_turn_item(
5555        &self,
5556        thread_id: &ThreadId,
5557        turn_id: &TurnId,
5558        item: &TranscriptItem,
5559    ) -> anyhow::Result<()> {
5560        let item_index = self.next_transcript_item_index(thread_id, turn_id).await?;
5561        let timestamp = OffsetDateTime::now_utc();
5562        self.emit(RoderEvent::TranscriptItemAppended(TranscriptItemAppended {
5563            thread_id: thread_id.clone(),
5564            turn_id: turn_id.clone(),
5565            item_type: match item {
5566                TranscriptItem::UserMessage(_) => "user_message",
5567                TranscriptItem::AssistantMessage(_) => "assistant_message",
5568                TranscriptItem::ReasoningSummary(_) => "reasoning_summary",
5569                TranscriptItem::ToolCall(_) => "tool_call",
5570                TranscriptItem::ToolResult(_) => "tool_result",
5571                TranscriptItem::FileChange(_) => "file_change",
5572                TranscriptItem::ContextCompaction(_) => "context_compaction",
5573                TranscriptItem::Error(_) => "error",
5574                TranscriptItem::ProviderMetadata(_) => "provider_metadata",
5575            }
5576            .to_string(),
5577            item_index: Some(item_index),
5578            item: Some(item.clone()),
5579            timestamp,
5580        }))
5581        .await;
5582        self.remember_transcript_item_index(thread_id, turn_id, item_index)
5583            .await?;
5584        Ok(())
5585    }
5586}
5587
5588fn transcript_has_images(transcript: &[TranscriptItem]) -> bool {
5589    transcript.iter().any(|item| {
5590        matches!(
5591            item,
5592            TranscriptItem::UserMessage(message) if !message.images.is_empty()
5593        )
5594    })
5595}
5596
5597fn transcript_has_task_ledger(transcript: &[TranscriptItem]) -> bool {
5598    transcript.iter().any(|item| {
5599        matches!(
5600            item,
5601            TranscriptItem::ToolResult(result)
5602                if result.name.as_deref() == Some(TASK_LEDGER_TOOL_NAME) && !result.is_error
5603        )
5604    })
5605}
5606
5607fn task_ledger_completion_prompt(transcript: &[TranscriptItem]) -> Option<String> {
5608    let latest = transcript.iter().rev().find_map(|item| match item {
5609        TranscriptItem::ToolResult(result)
5610            if result.name.as_deref() == Some(TASK_LEDGER_TOOL_NAME) && !result.is_error =>
5611        {
5612            Some(result.result.as_str())
5613        }
5614        _ => None,
5615    })?;
5616    if !task_ledger_has_open_items(latest) {
5617        return None;
5618    }
5619
5620    let mut ledger = latest.chars().take(1500).collect::<String>();
5621    if latest.chars().nth(1500).is_some() {
5622        ledger.push_str("...");
5623    }
5624    Some(format!(
5625        "Task Ledger Completion Required: the latest task ledger still has pending or in-progress items. Do not provide a final answer yet. Use tools to complete the remaining scoreable work, create or update any required output files, then call `{TASK_LEDGER_TOOL_NAME}` with every task completed and evidence before finalizing.\n\nLatest ledger:\n{ledger}"
5626    ))
5627}
5628
5629fn task_ledger_deadline_completion_prompt(
5630    remaining_seconds: u64,
5631    reserve_seconds: u64,
5632    completion_prompt: &str,
5633) -> String {
5634    format!(
5635        "Eval deadline scoreable completion: {remaining_seconds} seconds remain in the {reserve_seconds}-second finalization reserve. Do not browse, search, or start slow work. Use the available tools now to create or update the required scoreable output files, run only a quick local check if needed, then update the task ledger to completed before finalizing.\n\n{completion_prompt}"
5636    )
5637}
5638
5639fn task_ledger_scoreable_checkpoint_prompt(
5640    remaining_seconds: u64,
5641    completion_prompt: &str,
5642) -> String {
5643    format!(
5644        "Scoreable Output Checkpoint: {remaining_seconds} seconds remain before the eval deadline. Before any further research, browsing, or long commands, use tools now to ensure the required output file(s) exist with the best evidence-backed answer, even if provisional. If a scoreable file already exists, read it and preserve that candidate unless you have stronger task-specific evidence for a replacement. Do not overwrite a plausible dated, historical, or local-evidence candidate with a current live-page, partial-coverage, or weaker guess merely to refresh the checkpoint. You may continue refining afterward, but do not apologize or finalize until the scoreable file exists and the task ledger is updated.\n\n{completion_prompt}"
5645    )
5646}
5647
5648fn task_ledger_has_open_items(ledger: &str) -> bool {
5649    ledger.lines().any(|line| {
5650        let line = line.trim_start();
5651        line.starts_with("- pending:") || line.starts_with("- in_progress:")
5652    })
5653}
5654
5655fn turn_deadline_for_config(cfg: &RuntimeConfig) -> Option<OffsetDateTime> {
5656    cfg.turn_deadline_seconds
5657        .filter(|seconds| *seconds > 0)
5658        .map(|seconds| OffsetDateTime::now_utc() + Duration::seconds(seconds as i64))
5659}
5660
5661fn deadline_expired(deadline: OffsetDateTime) -> bool {
5662    OffsetDateTime::now_utc() >= deadline
5663}
5664
5665pub(crate) fn deadline_remaining_seconds(deadline: Option<OffsetDateTime>) -> Option<u64> {
5666    let deadline = deadline?;
5667    if deadline <= OffsetDateTime::now_utc() {
5668        return Some(0);
5669    }
5670    Some(
5671        (deadline - OffsetDateTime::now_utc())
5672            .unsigned_abs()
5673            .as_secs()
5674            .max(1),
5675    )
5676}
5677
5678fn deadline_instant(deadline: OffsetDateTime) -> tokio::time::Instant {
5679    let now = OffsetDateTime::now_utc();
5680    if deadline <= now {
5681        return tokio::time::Instant::now();
5682    }
5683    tokio::time::Instant::now() + (deadline - now).unsigned_abs()
5684}
5685
5686fn inference_timeout_deadline(
5687    deadline: Option<OffsetDateTime>,
5688    runtime_profile: RuntimeProfile,
5689    task_ledger_required: bool,
5690    reserve_seconds: u64,
5691    finalization_requested: bool,
5692    task_ledger_scoreable_checkpoints: u8,
5693    transcript: &[TranscriptItem],
5694) -> Option<(OffsetDateTime, InferenceTimeoutAction)> {
5695    let deadline = deadline?;
5696    if runtime_profile == RuntimeProfile::Eval
5697        && task_ledger_required
5698        && !finalization_requested
5699        && task_ledger_scoreable_checkpoints < TASK_LEDGER_SCOREABLE_CHECKPOINT_LIMIT
5700        && TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS > reserve_seconds
5701        && task_ledger_completion_prompt(transcript).is_some()
5702    {
5703        let checkpoint_deadline =
5704            deadline - Duration::seconds(TASK_LEDGER_SCOREABLE_CHECKPOINT_SECONDS as i64);
5705        if checkpoint_deadline > OffsetDateTime::now_utc() {
5706            return Some((
5707                checkpoint_deadline,
5708                InferenceTimeoutAction::ScoreableCheckpoint,
5709            ));
5710        }
5711    }
5712    if runtime_profile == RuntimeProfile::Eval && !finalization_requested {
5713        return Some((
5714            deadline - Duration::seconds(reserve_seconds as i64),
5715            InferenceTimeoutAction::Finalization,
5716        ));
5717    }
5718    Some((deadline, InferenceTimeoutAction::Finalization))
5719}
5720
5721fn turn_partial_result(transcript: &[TranscriptItem]) -> String {
5722    let tool_results = transcript
5723        .iter()
5724        .filter(|item| matches!(item, TranscriptItem::ToolResult(_)))
5725        .count();
5726    let assistant_messages = transcript
5727        .iter()
5728        .filter(|item| matches!(item, TranscriptItem::AssistantMessage(_)))
5729        .count();
5730    format!(
5731        "partial turn state: {} transcript items, {assistant_messages} assistant messages, {tool_results} tool results",
5732        transcript.len()
5733    )
5734}
5735
5736fn reasoning_for_model(cfg: &RuntimeConfig, model: &str) -> ReasoningConfig {
5737    let level = effective_reasoning_for_model(cfg, model);
5738    match level.as_str() {
5739        "" | REASONING_NONE => ReasoningConfig::default(),
5740        level => ReasoningConfig {
5741            enabled: true,
5742            level: Some(level.to_string()),
5743        },
5744    }
5745}
5746
5747fn server_side_compaction_threshold(cfg: &RuntimeConfig, model: &str) -> Option<u32> {
5748    let entry = lookup_model(model)?;
5749    if !entry.supports_compaction {
5750        return None;
5751    }
5752    cfg.auto_compact_token_limit
5753        .or_else(|| {
5754            model_profile_for_model(cfg, model).and_then(|profile| profile.auto_compact_token_limit)
5755        })
5756        .or(Some(entry.auto_compact_token_limit))
5757        .filter(|threshold| *threshold > 0)
5758}
5759
5760pub(crate) fn tool_search_for_provider_model(
5761    cfg: &RuntimeConfig,
5762    provider: &str,
5763    model: &str,
5764) -> ToolSearchConfig {
5765    let mut resolved = cfg.tool_search.clone();
5766    if let Some(provider_config) = cfg.provider_tool_search.get(provider) {
5767        provider_config.apply_to(&mut resolved);
5768    }
5769    if let Some(model_config) = cfg.model_tool_search.get(model) {
5770        model_config.apply_to(&mut resolved);
5771    }
5772    resolved
5773}
5774
5775fn parallel_tool_calls_for_model(cfg: &RuntimeConfig, model: &str) -> bool {
5776    cfg.model_parallel_tool_calls
5777        .get(model)
5778        .copied()
5779        .or_else(|| {
5780            model_profile_for_model(cfg, model).and_then(|profile| profile.parallel_tool_calls)
5781        })
5782        .unwrap_or(true)
5783}
5784
5785/// Count the children an `agent_swarm` call will launch from its arguments
5786/// (item-based spawns plus resumes). Lenient: returns 0 on malformed input.
5787fn agent_swarm_child_count(arguments: &str) -> usize {
5788    serde_json::from_str::<roder_api::subagents::AgentSwarmRequest>(arguments)
5789        .map(|request| request.items.len() + request.resume_agent_ids.len())
5790        .unwrap_or(0)
5791}
5792
5793/// Parse `completed`/`failed`/`aborted` counts from an `<agent_swarm_result>`
5794/// summary line, e.g. `<summary>completed: 2, failed: 1</summary>`. Omitted
5795/// buckets default to 0. Returns `None` when the text is not a swarm result.
5796fn parse_swarm_counts(text: &str) -> Option<(usize, usize, usize)> {
5797    if !text.contains("<agent_swarm_result>") {
5798        return None;
5799    }
5800    let bucket = |label: &str| -> usize {
5801        let needle = format!("{label}: ");
5802        text.find(&needle)
5803            .map(|start| start + needle.len())
5804            .map(|start| {
5805                text[start..]
5806                    .chars()
5807                    .take_while(|c| c.is_ascii_digit())
5808                    .collect::<String>()
5809            })
5810            .and_then(|digits| digits.parse().ok())
5811            .unwrap_or(0)
5812    };
5813    Some((bucket("completed"), bucket("failed"), bucket("aborted")))
5814}
5815
5816fn effective_reasoning_for_model(cfg: &RuntimeConfig, model: &str) -> String {
5817    let base_reasoning = default_effective_reasoning_for_model(cfg, model);
5818    if cfg.dynamic_workflows.effort_profile == DynamicWorkflowEffortProfile::Ultracode {
5819        return ultracode_reasoning_level_for_model(
5820            model,
5821            &cfg.speed_policy.ultracode_reasoning,
5822            &base_reasoning,
5823        );
5824    }
5825    base_reasoning
5826}
5827
5828fn default_effective_reasoning_for_model(cfg: &RuntimeConfig, model: &str) -> String {
5829    let Some(entry) = lookup_model(model) else {
5830        return cfg
5831            .reasoning
5832            .clone()
5833            .unwrap_or_else(|| REASONING_NONE.to_string());
5834    };
5835    if entry.supported_reasoning.is_empty() {
5836        return REASONING_NONE.to_string();
5837    }
5838    cfg.reasoning
5839        .as_deref()
5840        .filter(|reasoning| {
5841            entry
5842                .supported_reasoning
5843                .iter()
5844                .any(|option| option.effort == *reasoning)
5845        })
5846        .map(str::to_string)
5847        .or_else(|| {
5848            model_profile_for_model(cfg, model)
5849                .and_then(|profile| profile.reasoning.orientation)
5850                .filter(|reasoning| {
5851                    entry
5852                        .supported_reasoning
5853                        .iter()
5854                        .any(|option| option.effort == reasoning)
5855                })
5856        })
5857        .unwrap_or_else(|| entry.default_reasoning.to_string())
5858}
5859
5860fn validate_reasoning_effort(model: &str, effort: &str) -> anyhow::Result<()> {
5861    if effort == REASONING_NONE && !model_supports_reasoning(model, effort) {
5862        return Ok(());
5863    }
5864    let Some(entry) = lookup_model(model) else {
5865        return Ok(());
5866    };
5867    if entry
5868        .supported_reasoning
5869        .iter()
5870        .any(|option| option.effort == effort)
5871    {
5872        Ok(())
5873    } else {
5874        anyhow::bail!("model {model} does not support reasoning effort {effort}")
5875    }
5876}
5877
5878fn validate_runtime_config_reasoning(cfg: &RuntimeConfig) -> anyhow::Result<()> {
5879    let Some(reasoning) = cfg.reasoning.as_deref() else {
5880        return Ok(());
5881    };
5882    let Some(entry) = lookup_model(&cfg.default_model) else {
5883        return Ok(());
5884    };
5885    if entry.provider != PROVIDER_GEMINI {
5886        return Ok(());
5887    }
5888    validate_reasoning_effort(&cfg.default_model, reasoning)
5889}
5890
5891fn validate_runtime_inference_router_config(
5892    registry: &ExtensionRegistry,
5893    cfg: &RuntimeConfig,
5894) -> anyhow::Result<()> {
5895    if !cfg.inference_router.enabled {
5896        return Ok(());
5897    }
5898    let Some(router_id) = cfg.inference_router.router_id.as_deref() else {
5899        anyhow::bail!("inference_router.enabled requires inference_router.router");
5900    };
5901    if registry.inference_router(router_id).is_some() {
5902        return Ok(());
5903    }
5904    let available = registry
5905        .inference_routers
5906        .iter()
5907        .map(|router| router.id())
5908        .collect::<Vec<_>>()
5909        .join(", ");
5910    if available.is_empty() {
5911        anyhow::bail!("inference router {router_id:?} is not registered");
5912    }
5913    anyhow::bail!(
5914        "inference router {router_id:?} is not registered; available routers: {available}"
5915    );
5916}
5917
5918fn model_supports_reasoning(model: &str, effort: &str) -> bool {
5919    lookup_model(model)
5920        .map(|entry| {
5921            entry
5922                .supported_reasoning
5923                .iter()
5924                .any(|option| option.effort == effort)
5925        })
5926        .unwrap_or(false)
5927}
5928
5929fn is_final_answer_phase(phase: Option<&str>) -> bool {
5930    phase.is_none_or(|phase| phase.is_empty() || phase == FINAL_ANSWER_PHASE)
5931}
5932
5933fn edit_tool_for_model<'a>(cfg: &'a RuntimeConfig, model: &'a str) -> Option<&'a str> {
5934    cfg.model_edit_tools
5935        .get(model)
5936        .map(String::as_str)
5937        .or_else(|| {
5938            cfg.model_profiles
5939                .get(model)
5940                .and_then(|profile| profile.edit_tool.as_deref())
5941        })
5942        .or_else(|| lookup_model(model).and_then(|entry| entry.edit_tool))
5943        .or(Some(EDIT_TOOL_EDIT))
5944}
5945
5946fn model_profile_for_model(cfg: &RuntimeConfig, model: &str) -> Option<ModelHarnessProfile> {
5947    cfg.model_profiles
5948        .get(model)
5949        .cloned()
5950        .or_else(|| built_in_model_profile(model))
5951}
5952
5953pub(crate) fn allowlist_permits(allowlist: &[String], tool_name: &str) -> bool {
5954    allowlist.is_empty() || allowlist.iter().any(|allowed| allowed == tool_name)
5955}
5956
5957/// Provider-aware profile resolution for the active turn.
5958///
5959/// Many model ids are shared across providers (for example Cursor proxies
5960/// `claude-opus-4-8`). Resolving the harness profile by id alone picks the
5961/// first catalog entry, which assigns cross-provider ids the wrong family and
5962/// instruction overlay (e.g. a `cursor/claude-opus-4-8` turn would otherwise
5963/// inherit the Anthropic overlay). Prefer the explicit `(provider, model)`
5964/// catalog entry, keeping user-configured profile overrides as the top
5965/// precedence.
5966fn model_profile_for_provider_model(
5967    cfg: &RuntimeConfig,
5968    provider: &str,
5969    model: &str,
5970) -> Option<ModelHarnessProfile> {
5971    cfg.model_profiles
5972        .get(model)
5973        .cloned()
5974        .or_else(|| built_in_model_profile_for_provider(provider, model))
5975}
5976
5977fn schema_policy_for_model(profile: Option<&ModelHarnessProfile>) -> ModelSchemaPolicy {
5978    profile
5979        .map(|profile| profile.schema_policy)
5980        .unwrap_or_default()
5981}
5982
5983fn model_profile_segment_metadata(
5984    profile: Option<&ModelHarnessProfile>,
5985    provider: &str,
5986    model: &str,
5987    segment: &str,
5988) -> serde_json::Value {
5989    serde_json::json!({
5990        "kind": MODEL_PROFILE_TRACE_KIND,
5991        "segment": segment,
5992        "provider": provider,
5993        "model": model,
5994        "profileModel": profile.map(|profile| profile.model.as_str()).unwrap_or(model),
5995        "providerFamily": profile.map(|profile| profile.provider_family),
5996        "editTool": profile.and_then(|profile| profile.edit_tool.as_deref()),
5997        "schemaPolicy": profile.map(|profile| profile.schema_policy),
5998        "instructionOverlay": profile.map(|profile| profile.instruction_overlay),
5999        "parallelToolCalls": profile.and_then(|profile| profile.parallel_tool_calls),
6000        "autoCompactTokenLimit": profile.and_then(|profile| profile.auto_compact_token_limit),
6001    })
6002}
6003
6004fn model_switch_summary(
6005    transcript: &[TranscriptItem],
6006    profile: Option<&ModelHarnessProfile>,
6007    provider: &str,
6008    model: &str,
6009    tools: &[roder_api::tools::ToolSpec],
6010) -> Option<String> {
6011    let previous = latest_model_profile_segment(transcript)?;
6012    let previous_model = previous
6013        .get("model")
6014        .and_then(serde_json::Value::as_str)
6015        .unwrap_or_default();
6016    let previous_provider = previous
6017        .get("provider")
6018        .and_then(serde_json::Value::as_str)
6019        .unwrap_or_default();
6020    if previous_model == model && previous_provider == provider {
6021        return None;
6022    }
6023
6024    let previous_profile = previous
6025        .get("profileModel")
6026        .and_then(serde_json::Value::as_str)
6027        .unwrap_or(previous_model);
6028    let current_profile = profile
6029        .map(|profile| profile.model.as_str())
6030        .unwrap_or(model);
6031    let previous_edit_tool = previous
6032        .get("editTool")
6033        .and_then(serde_json::Value::as_str)
6034        .unwrap_or("none");
6035    let current_edit_tool = profile
6036        .and_then(|profile| profile.edit_tool.as_deref())
6037        .unwrap_or("none");
6038    let tool_names = tools
6039        .iter()
6040        .map(|tool| tool.name.as_str())
6041        .take(12)
6042        .collect::<Vec<_>>()
6043        .join(", ");
6044    Some(format!(
6045        "{MODEL_SWITCH_SUMMARY_PREFIX} previous profile {previous_provider}/{previous_profile} used edit tool {previous_edit_tool}. Current profile {provider}/{current_profile} uses edit tool {current_edit_tool}. Available tools now: {}.",
6046        if tool_names.is_empty() {
6047            "none"
6048        } else {
6049            &tool_names
6050        }
6051    ))
6052}
6053
6054fn latest_model_profile_segment(transcript: &[TranscriptItem]) -> Option<&serde_json::Value> {
6055    transcript.iter().rev().find_map(|item| {
6056        let TranscriptItem::ProviderMetadata(value) = item else {
6057            return None;
6058        };
6059        (value.get("kind").and_then(serde_json::Value::as_str) == Some(MODEL_PROFILE_TRACE_KIND))
6060            .then_some(value)
6061    })
6062}
6063
6064pub fn validate_edit_tool(value: &str) -> anyhow::Result<()> {
6065    match value.trim() {
6066        EDIT_TOOL_PATCH | EDIT_TOOL_EDIT => Ok(()),
6067        _ => anyhow::bail!(
6068            "unsupported edit_tool {value:?}; allowed values: {EDIT_TOOL_PATCH}, {EDIT_TOOL_EDIT}"
6069        ),
6070    }
6071}
6072
6073fn should_persist_thread_event(thread_id: &str) -> bool {
6074    !is_synthetic_event_thread_id(thread_id)
6075}
6076
6077#[cfg(test)]
6078#[path = "runtime/item_identity_tests.rs"]
6079mod item_identity_tests;
6080
6081#[cfg(test)]
6082#[path = "runtime/codex_v2_tests.rs"]
6083mod codex_v2_tests;
6084
6085#[cfg(test)]
6086#[path = "runtime/codex_v2_lifecycle_tests.rs"]
6087mod codex_v2_lifecycle_tests;
6088
6089#[cfg(test)]
6090mod tests {
6091    use super::*;
6092    use futures::stream;
6093    use roder_api::catalog::{
6094        PROVIDER_MOCK, REASONING_HIGH, REASONING_LOW, REASONING_MEDIUM, REASONING_MINIMAL,
6095        REASONING_NONE, REASONING_XHIGH,
6096    };
6097    use roder_api::extension::ExtensionRegistryBuilder;
6098    use roder_api::inference::{
6099        CompletionMetadata, InferenceCapabilities, InferenceEngine, InferenceEventStream,
6100        InferenceProviderContext, InferenceTurnContext, MessageDelta, ModelDescriptor,
6101        ModelInstructionOverlay, ModelProfileReasoning, ModelSchemaPolicy, ProviderFamily,
6102        ReasoningEffortDescriptor,
6103    };
6104    use roder_api::inference_routing::{
6105        InferenceRouter, InferenceRoutingContext, InferenceRoutingDecision, InferenceRoutingOutcome,
6106    };
6107    use roder_api::thread::ThreadStoreFactory;
6108    use roder_api::tools::{ToolContributor, ToolExecutor, ToolSpec};
6109    use roder_ext_jsonl_thread_store::store::JsonlThreadStoreFactory;
6110    use std::sync::Mutex as StdMutex;
6111
6112    fn test_workspace() -> String {
6113        std::env::current_dir().unwrap().display().to_string()
6114    }
6115
6116    #[test]
6117    fn interactive_explicit_turn_deadline_has_bounded_remaining_time() {
6118        let config = RuntimeConfig {
6119            runtime_profile: RuntimeProfile::Interactive,
6120            turn_deadline_seconds: Some(60),
6121            ..RuntimeConfig::default()
6122        };
6123
6124        let deadline = turn_deadline_for_config(&config).expect("configured turn deadline");
6125        let remaining = deadline_remaining_seconds(Some(deadline));
6126
6127        assert!(
6128            matches!(remaining, Some(1..=60)),
6129            "interactive deadline must be positive and no longer than configured: {remaining:?}"
6130        );
6131    }
6132
6133    #[test]
6134    fn interactive_without_turn_deadline_stays_unbounded() {
6135        let config = RuntimeConfig {
6136            runtime_profile: RuntimeProfile::Interactive,
6137            turn_deadline_seconds: None,
6138            ..RuntimeConfig::default()
6139        };
6140
6141        assert_eq!(turn_deadline_for_config(&config), None);
6142    }
6143
6144    struct MetadataMissingStore;
6145
6146    #[async_trait::async_trait]
6147    impl ThreadStore for MetadataMissingStore {
6148        fn id(&self) -> roder_api::thread::ThreadStoreId {
6149            "metadata-missing-store".to_string()
6150        }
6151
6152        async fn create_thread(&self, metadata: ThreadMetadata) -> anyhow::Result<ThreadMetadata> {
6153            Ok(metadata)
6154        }
6155
6156        async fn list_threads(&self) -> anyhow::Result<Vec<ThreadMetadata>> {
6157            Ok(Vec::new())
6158        }
6159
6160        async fn load_thread(
6161            &self,
6162            _thread_id: &ThreadId,
6163        ) -> anyhow::Result<Option<ThreadSnapshot>> {
6164            Ok(Some(ThreadSnapshot {
6165                metadata: None,
6166                ..ThreadSnapshot::default()
6167            }))
6168        }
6169
6170        async fn append_event(
6171            &self,
6172            _thread_id: &ThreadId,
6173            _envelope: &EventEnvelope,
6174        ) -> anyhow::Result<()> {
6175            Ok(())
6176        }
6177    }
6178
6179    struct MetadataMissingStoreFactory;
6180
6181    impl ThreadStoreFactory for MetadataMissingStoreFactory {
6182        fn id(&self) -> roder_api::thread::ThreadStoreId {
6183            "metadata-missing-store".to_string()
6184        }
6185
6186        fn create(&self) -> Arc<dyn ThreadStore> {
6187            Arc::new(MetadataMissingStore)
6188        }
6189    }
6190
6191    #[test]
6192    fn synthetic_app_server_events_are_not_thread_events() {
6193        for thread_id in ["app-server", "runtime", "thread-workflow"] {
6194            assert!(!should_persist_thread_event(thread_id));
6195        }
6196        assert!(should_persist_thread_event("thread-discovery"));
6197        assert!(should_persist_thread_event("thread-plan"));
6198        assert!(should_persist_thread_event("thread-process"));
6199        assert!(should_persist_thread_event("thread-1"));
6200    }
6201
6202    #[test]
6203    fn server_side_compaction_uses_catalog_ninety_percent_default() {
6204        assert_eq!(
6205            server_side_compaction_threshold(&RuntimeConfig::default(), "gpt-5.5"),
6206            Some(945_000)
6207        );
6208        assert_eq!(
6209            server_side_compaction_threshold(&RuntimeConfig::default(), "gpt-5.3-codex-spark"),
6210            Some(115_200)
6211        );
6212    }
6213
6214    #[test]
6215    fn server_side_compaction_respects_explicit_config_override() {
6216        let cfg = RuntimeConfig {
6217            auto_compact_token_limit: Some(123_456),
6218            ..RuntimeConfig::default()
6219        };
6220
6221        assert_eq!(
6222            server_side_compaction_threshold(&cfg, "gpt-5.5"),
6223            Some(123_456)
6224        );
6225    }
6226
6227    #[tokio::test]
6228    async fn pre_request_compaction_runs_when_server_side_model_is_at_context_window() {
6229        let captured = Arc::new(StdMutex::new(None));
6230        let mut builder = ExtensionRegistryBuilder::new();
6231        builder.inference_engine(Arc::new(CapturingEngine {
6232            request: captured.clone(),
6233        }));
6234        let thread_root = std::env::temp_dir().join(format!(
6235            "roder-pre-request-compaction-{}",
6236            uuid::Uuid::new_v4()
6237        ));
6238        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
6239            base_path: thread_root.clone(),
6240        }));
6241        let runtime = Arc::new(
6242            Runtime::new(
6243                builder.build().unwrap(),
6244                RuntimeConfig {
6245                    default_provider: PROVIDER_MOCK.to_string(),
6246                    default_model: "gpt-5.5".to_string(),
6247                    file_backed_dynamic_context: true,
6248                    ..RuntimeConfig::default()
6249                },
6250            )
6251            .unwrap(),
6252        );
6253        let thread_id = runtime
6254            .create_thread(Some("Pre-request compaction".to_string()))
6255            .await
6256            .unwrap()
6257            .thread_id;
6258        let old_turn = "old-turn".to_string();
6259        runtime
6260            .persist_turn_item(
6261                &thread_id,
6262                &old_turn,
6263                &TranscriptItem::UserMessage(UserMessage::text("old context ".repeat(4_300_000))),
6264            )
6265            .await
6266            .unwrap();
6267
6268        let mut events = runtime.subscribe_events();
6269        runtime
6270            .start_turn(StartTurnRequest {
6271                thread_id: thread_id.clone(),
6272                message: "continue".to_string(),
6273                images: Vec::new(),
6274                provider_override: None,
6275                model_override: None,
6276                reasoning_override: None,
6277                workspace: test_workspace(),
6278                instructions: InstructionBundle::default(),
6279                developer_context: None,
6280                task_ledger_required: false,
6281                service_tier_override: None,
6282            })
6283            .await
6284            .unwrap();
6285        loop {
6286            let envelope = tokio::time::timeout(std::time::Duration::from_secs(5), events.recv())
6287                .await
6288                .unwrap()
6289                .unwrap();
6290            if envelope.thread_id.as_deref() == Some(&thread_id)
6291                && matches!(envelope.event, RoderEvent::TurnCompleted(_))
6292            {
6293                break;
6294            }
6295        }
6296
6297        let request = captured.lock().unwrap().clone().unwrap();
6298        assert!(
6299            matches!(
6300                request.transcript.first(),
6301                Some(TranscriptItem::ContextCompaction(_))
6302            ),
6303            "provider request should start with a local emergency compaction item"
6304        );
6305        assert!(
6306            request.transcript.len() < 4,
6307            "provider request should not replay the full oversized prior transcript: {:?}",
6308            request.transcript
6309        );
6310
6311        let _ = std::fs::remove_dir_all(thread_root);
6312    }
6313
6314    #[tokio::test]
6315    async fn continue_after_context_window_failure_compacts_before_provider_request() {
6316        let captured = Arc::new(StdMutex::new(None));
6317        let mut builder = ExtensionRegistryBuilder::new();
6318        builder.inference_engine(Arc::new(CapturingEngine {
6319            request: captured.clone(),
6320        }));
6321        let thread_root = std::env::temp_dir().join(format!(
6322            "roder-context-failure-continue-{}",
6323            uuid::Uuid::new_v4()
6324        ));
6325        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
6326            base_path: thread_root.clone(),
6327        }));
6328        let runtime = Arc::new(
6329            Runtime::new(
6330                builder.build().unwrap(),
6331                RuntimeConfig {
6332                    default_provider: PROVIDER_MOCK.to_string(),
6333                    default_model: "gpt-5.5".to_string(),
6334                    file_backed_dynamic_context: true,
6335                    ..RuntimeConfig::default()
6336                },
6337            )
6338            .unwrap(),
6339        );
6340        let thread_id = runtime
6341            .create_thread(Some("Context failure continue".to_string()))
6342            .await
6343            .unwrap()
6344            .thread_id;
6345        let failed_turn = "failed-turn".to_string();
6346        runtime
6347            .persist_turn_item(
6348                &thread_id,
6349                &failed_turn,
6350                &TranscriptItem::UserMessage(UserMessage::text("old work ".repeat(10_000))),
6351            )
6352            .await
6353            .unwrap();
6354        runtime
6355            .persist_turn_item(
6356                &thread_id,
6357                &failed_turn,
6358                &TranscriptItem::Error(ErrorRecord {
6359                    message: "Your input exceeds the context window of this model. Please adjust your input and try again."
6360                        .to_string(),
6361                }),
6362            )
6363            .await
6364            .unwrap();
6365
6366        let mut events = runtime.subscribe_events();
6367        runtime
6368            .start_turn(StartTurnRequest {
6369                thread_id: thread_id.clone(),
6370                message: "continue".to_string(),
6371                images: Vec::new(),
6372                provider_override: None,
6373                model_override: None,
6374                reasoning_override: None,
6375                workspace: test_workspace(),
6376                instructions: InstructionBundle::default(),
6377                developer_context: None,
6378                task_ledger_required: false,
6379                service_tier_override: None,
6380            })
6381            .await
6382            .unwrap();
6383        loop {
6384            let envelope = tokio::time::timeout(std::time::Duration::from_secs(5), events.recv())
6385                .await
6386                .unwrap()
6387                .unwrap();
6388            if envelope.thread_id.as_deref() == Some(&thread_id)
6389                && matches!(envelope.event, RoderEvent::TurnCompleted(_))
6390            {
6391                break;
6392            }
6393        }
6394
6395        let request = captured.lock().unwrap().clone().unwrap();
6396        assert!(
6397            matches!(
6398                request.transcript.first(),
6399                Some(TranscriptItem::ContextCompaction(_))
6400            ),
6401            "provider request after context-window failure should start with local compaction"
6402        );
6403        assert!(
6404            request
6405                .transcript
6406                .iter()
6407                .any(|item| matches!(item, TranscriptItem::UserMessage(message) if message.text == "continue")),
6408            "current continue prompt must be preserved: {:?}",
6409            request.transcript
6410        );
6411        assert!(
6412            !request.transcript.iter().any(
6413                |item| matches!(item, TranscriptItem::Error(error) if error.message.contains("context window"))
6414            ),
6415            "raw prior context-window error should be summarized, not replayed: {:?}",
6416            request.transcript
6417        );
6418
6419        let _ = std::fs::remove_dir_all(thread_root);
6420    }
6421
6422    #[tokio::test]
6423    async fn workspace_for_thread_falls_back_when_metadata_is_missing() {
6424        let workspace = test_workspace();
6425        let mut builder = ExtensionRegistryBuilder::new();
6426        builder.inference_engine(Arc::new(FakeInferenceEngine));
6427        builder.thread_store_factory(Arc::new(MetadataMissingStoreFactory));
6428        let runtime = Runtime::new(
6429            builder.build().unwrap(),
6430            RuntimeConfig {
6431                workspace: Some(workspace.clone()),
6432                ..RuntimeConfig::default()
6433            },
6434        )
6435        .unwrap();
6436
6437        let resolved = runtime
6438            .workspace_for_thread(&ThreadId::from("thread-workflow"))
6439            .await
6440            .unwrap();
6441
6442        assert_eq!(resolved, workspace);
6443    }
6444
6445    #[tokio::test]
6446    async fn automations_can_create_project_thread_with_model_overrides() {
6447        let runtime = Runtime::fake().unwrap();
6448        let workspace = std::env::temp_dir().join("project");
6449        let metadata = runtime
6450            .create_thread_with(CreateThreadRequest {
6451                title: Some("Automation: nightly status".to_string()),
6452                workspace: workspace.display().to_string(),
6453                workspace_id: None,
6454                root_id: None,
6455                provider: Some("mock".to_string()),
6456                model: Some("mock".to_string()),
6457                selection_mode: None,
6458                tool_allowlist: Vec::new(),
6459                service_tier: None,
6460                developer_instructions: None,
6461                external_tools: Vec::new(),
6462                runner: None,
6463            })
6464            .await
6465            .unwrap();
6466
6467        assert_eq!(
6468            metadata.title.as_deref(),
6469            Some("Automation: nightly status")
6470        );
6471        assert_eq!(metadata.workspace, workspace.display().to_string());
6472        assert_eq!(metadata.provider.as_deref(), Some("mock"));
6473        assert_eq!(metadata.model.as_deref(), Some("mock"));
6474    }
6475
6476    #[tokio::test]
6477    async fn prior_provider_compaction_boundary_is_used_on_next_turn() {
6478        let captured = Arc::new(StdMutex::new(None));
6479        let mut builder = ExtensionRegistryBuilder::new();
6480        builder.inference_engine(Arc::new(CapturingEngine {
6481            request: captured.clone(),
6482        }));
6483        let thread_root = std::env::temp_dir().join(format!(
6484            "roder-provider-compaction-boundary-{}",
6485            uuid::Uuid::new_v4()
6486        ));
6487        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
6488            base_path: thread_root.clone(),
6489        }));
6490        let runtime = Arc::new(
6491            Runtime::new(
6492                builder.build().unwrap(),
6493                RuntimeConfig {
6494                    default_provider: PROVIDER_MOCK.to_string(),
6495                    default_model: "gpt-5.5".to_string(),
6496                    ..RuntimeConfig::default()
6497                },
6498            )
6499            .unwrap(),
6500        );
6501        let thread_id = runtime
6502            .create_thread(Some("Provider compaction boundary".to_string()))
6503            .await
6504            .unwrap()
6505            .thread_id;
6506        let old_turn = "old-turn".to_string();
6507        runtime
6508            .persist_turn_item(
6509                &thread_id,
6510                &old_turn,
6511                &TranscriptItem::UserMessage(UserMessage::text(
6512                    "old history that must not be replayed after provider compaction",
6513                )),
6514            )
6515            .await
6516            .unwrap();
6517        runtime
6518            .persist_turn_item(
6519                &thread_id,
6520                &old_turn,
6521                &TranscriptItem::AssistantMessage(AssistantMessage {
6522                    text: "old answer".to_string(),
6523                    phase: None,
6524                }),
6525            )
6526            .await
6527            .unwrap();
6528        runtime
6529            .persist_turn_item(
6530                &thread_id,
6531                &old_turn,
6532                &TranscriptItem::ProviderMetadata(serde_json::json!({
6533                    "output": [{
6534                        "id": "cmp_1",
6535                        "type": "compaction",
6536                        "encrypted_content": "opaque-state"
6537                    }]
6538                })),
6539            )
6540            .await
6541            .unwrap();
6542        runtime
6543            .persist_turn_item(
6544                &thread_id,
6545                &old_turn,
6546                &TranscriptItem::AssistantMessage(AssistantMessage {
6547                    text: "after compact".to_string(),
6548                    phase: None,
6549                }),
6550            )
6551            .await
6552            .unwrap();
6553
6554        let mut events = runtime.subscribe_events();
6555        runtime
6556            .start_turn(StartTurnRequest {
6557                thread_id: thread_id.clone(),
6558                message: "continue".to_string(),
6559                images: Vec::new(),
6560                provider_override: None,
6561                model_override: None,
6562                reasoning_override: None,
6563                workspace: test_workspace(),
6564                instructions: InstructionBundle::default(),
6565                developer_context: None,
6566                task_ledger_required: false,
6567                service_tier_override: None,
6568            })
6569            .await
6570            .unwrap();
6571        loop {
6572            let envelope = tokio::time::timeout(std::time::Duration::from_secs(5), events.recv())
6573                .await
6574                .unwrap()
6575                .unwrap();
6576            if envelope.thread_id.as_deref() == Some(&thread_id)
6577                && matches!(envelope.event, RoderEvent::TurnCompleted(_))
6578            {
6579                break;
6580            }
6581        }
6582
6583        let request = captured.lock().unwrap().clone().unwrap();
6584        let compaction_idx = request.transcript.iter().position(|item| {
6585            matches!(
6586                item,
6587                TranscriptItem::ProviderMetadata(metadata)
6588                    if crate::compaction::provider_metadata_has_compaction(metadata)
6589            )
6590        });
6591        assert!(
6592            compaction_idx.is_some(),
6593            "next provider request should include the provider compaction item: {:?}",
6594            request.transcript
6595        );
6596        // Skill/context injectors may prepend non-history items, but no prior-turn
6597        // conversation may appear before the latest provider compaction boundary.
6598        let history_before_boundary =
6599            request
6600                .transcript
6601                .iter()
6602                .take(compaction_idx.unwrap())
6603                .any(|item| match item {
6604                    TranscriptItem::AssistantMessage(_)
6605                    | TranscriptItem::ToolCall(_)
6606                    | TranscriptItem::ToolResult(_) => true,
6607                    TranscriptItem::UserMessage(message) => {
6608                        !message.text.contains("<skills>") && message.text != "continue"
6609                    }
6610                    _ => false,
6611                });
6612        assert!(
6613            !history_before_boundary,
6614            "pre-provider-compaction conversation must not be replayed: {:?}",
6615            request.transcript
6616        );
6617        assert!(
6618            !request.transcript.iter().any(|item| {
6619                matches!(
6620                    item,
6621                    TranscriptItem::UserMessage(message)
6622                        if message.text.contains("old history that must not be replayed")
6623                )
6624            }),
6625            "pre-provider-compaction history must not be replayed: {:?}",
6626            request.transcript
6627        );
6628        assert!(
6629            request.transcript.iter().any(|item| {
6630                matches!(
6631                    item,
6632                    TranscriptItem::UserMessage(message) if message.text == "continue"
6633                )
6634            }),
6635            "current continue prompt must be preserved: {:?}",
6636            request.transcript
6637        );
6638        assert_eq!(
6639            request.runtime.auto_compact_token_limit,
6640            Some(945_000),
6641            "gpt-5.5 should keep server-side auto compaction configured"
6642        );
6643
6644        let _ = std::fs::remove_dir_all(thread_root);
6645    }
6646
6647    #[test]
6648    fn server_side_compaction_is_only_enabled_for_supported_models() {
6649        let cfg = RuntimeConfig {
6650            auto_compact_token_limit: Some(123_456),
6651            ..RuntimeConfig::default()
6652        };
6653
6654        assert_eq!(server_side_compaction_threshold(&cfg, "mock"), None);
6655        assert_eq!(
6656            server_side_compaction_threshold(&cfg, "codex-auto-review"),
6657            None
6658        );
6659    }
6660
6661    #[test]
6662    fn reasoning_is_disabled_for_models_without_reasoning_support() {
6663        let cfg = RuntimeConfig {
6664            reasoning: Some(REASONING_HIGH.to_string()),
6665            ..RuntimeConfig::default()
6666        };
6667
6668        assert_eq!(
6669            effective_reasoning_for_model(&cfg, "claude-haiku-4-5-20251001"),
6670            REASONING_NONE
6671        );
6672        assert_eq!(
6673            reasoning_for_model(&cfg, "claude-haiku-4-5-20251001"),
6674            ReasoningConfig::default()
6675        );
6676    }
6677
6678    #[test]
6679    fn unsupported_configured_reasoning_falls_back_to_model_default() {
6680        let cfg = RuntimeConfig {
6681            reasoning: Some(REASONING_MINIMAL.to_string()),
6682            ..RuntimeConfig::default()
6683        };
6684
6685        assert_eq!(
6686            effective_reasoning_for_model(&cfg, "gpt-5.5"),
6687            REASONING_MEDIUM
6688        );
6689    }
6690
6691    #[test]
6692    fn unsupported_configured_gemini_reasoning_is_rejected() {
6693        let mut builder = ExtensionRegistryBuilder::new();
6694        builder.inference_engine(std::sync::Arc::new(FakeInferenceEngine));
6695
6696        let err = match Runtime::new(
6697            builder.build().unwrap(),
6698            RuntimeConfig {
6699                default_model: "gemini-3.5-flash".to_string(),
6700                reasoning: Some(REASONING_XHIGH.to_string()),
6701                ..RuntimeConfig::default()
6702            },
6703        ) {
6704            Ok(_) => panic!("expected unsupported Gemini reasoning to be rejected"),
6705            Err(err) => err,
6706        };
6707
6708        assert!(
6709            err.to_string()
6710                .contains("model gemini-3.5-flash does not support reasoning effort xhigh")
6711        );
6712    }
6713
6714    #[tokio::test]
6715    async fn selecting_none_for_non_reasoning_model_preserves_stored_preference() {
6716        let runtime = Runtime::new(
6717            Runtime::fake().unwrap().registry,
6718            RuntimeConfig {
6719                reasoning: Some(REASONING_HIGH.to_string()),
6720                ..RuntimeConfig::default()
6721            },
6722        )
6723        .unwrap();
6724
6725        let cfg = runtime
6726            .select_provider(
6727                roder_api::catalog::PROVIDER_MOCK.to_string(),
6728                Some("claude-haiku-4-5-20251001".to_string()),
6729                Some(REASONING_NONE.to_string()),
6730            )
6731            .await
6732            .unwrap();
6733
6734        assert_eq!(cfg.reasoning.as_deref(), Some(REASONING_HIGH));
6735        assert_eq!(runtime.effective_reasoning().await, REASONING_NONE);
6736    }
6737
6738    #[tokio::test]
6739    async fn selecting_none_for_model_that_supports_none_updates_preference() {
6740        let runtime = Runtime::new(
6741            Runtime::fake().unwrap().registry,
6742            RuntimeConfig {
6743                reasoning: Some(REASONING_HIGH.to_string()),
6744                ..RuntimeConfig::default()
6745            },
6746        )
6747        .unwrap();
6748
6749        let cfg = runtime
6750            .select_provider(
6751                roder_api::catalog::PROVIDER_MOCK.to_string(),
6752                Some("mock".to_string()),
6753                Some(REASONING_NONE.to_string()),
6754            )
6755            .await
6756            .unwrap();
6757
6758        assert_eq!(cfg.reasoning.as_deref(), Some(REASONING_NONE));
6759    }
6760
6761    #[test]
6762    fn parallel_tool_calls_default_on_with_model_override() {
6763        assert!(parallel_tool_calls_for_model(
6764            &RuntimeConfig::default(),
6765            "custom-model"
6766        ));
6767
6768        let cfg = RuntimeConfig {
6769            model_parallel_tool_calls: std::collections::HashMap::from([(
6770                "custom-model".to_string(),
6771                false,
6772            )]),
6773            ..RuntimeConfig::default()
6774        };
6775
6776        assert!(!parallel_tool_calls_for_model(&cfg, "custom-model"));
6777        assert!(parallel_tool_calls_for_model(&cfg, "other-model"));
6778    }
6779
6780    #[test]
6781    fn profile_parallel_tool_calls_applies_between_config_and_default() {
6782        let cfg = RuntimeConfig {
6783            model_profiles: std::collections::HashMap::from([(
6784                "gpt-5.5".to_string(),
6785                test_model_profile("gpt-5.5"),
6786            )]),
6787            ..RuntimeConfig::default()
6788        };
6789
6790        assert!(!parallel_tool_calls_for_model(&cfg, "gpt-5.5"));
6791
6792        let cfg = RuntimeConfig {
6793            model_parallel_tool_calls: std::collections::HashMap::from([(
6794                "gpt-5.5".to_string(),
6795                true,
6796            )]),
6797            ..cfg
6798        };
6799
6800        assert!(parallel_tool_calls_for_model(&cfg, "gpt-5.5"));
6801    }
6802
6803    struct CapturingEngine {
6804        request: Arc<StdMutex<Option<AgentInferenceRequest>>>,
6805    }
6806
6807    #[async_trait::async_trait]
6808    impl InferenceEngine for CapturingEngine {
6809        fn id(&self) -> String {
6810            roder_api::catalog::PROVIDER_MOCK.to_string()
6811        }
6812
6813        fn capabilities(&self) -> InferenceCapabilities {
6814            InferenceCapabilities::coding_agent_default()
6815        }
6816
6817        async fn list_models(
6818            &self,
6819            _ctx: InferenceProviderContext<'_>,
6820        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
6821            Ok(roder_api::catalog::models_for_provider(
6822                roder_api::catalog::PROVIDER_MOCK,
6823                true,
6824            ))
6825        }
6826
6827        async fn stream_turn(
6828            &self,
6829            _ctx: InferenceTurnContext<'_>,
6830            request: AgentInferenceRequest,
6831        ) -> anyhow::Result<InferenceEventStream> {
6832            *self.request.lock().unwrap() = Some(request);
6833            Ok(Box::pin(stream::iter(vec![
6834                Ok(InferenceEvent::MessageDelta(MessageDelta {
6835                    text: "done".to_string(),
6836                    phase: None,
6837                })),
6838                Ok(InferenceEvent::Completed(CompletionMetadata {
6839                    stop_reason: Some("stop".to_string()),
6840                    provider_response_id: None,
6841                })),
6842            ])))
6843        }
6844    }
6845
6846    struct RoutingCaptureEngine {
6847        id: &'static str,
6848        models: Vec<ModelDescriptor>,
6849        requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
6850    }
6851
6852    #[async_trait::async_trait]
6853    impl InferenceEngine for RoutingCaptureEngine {
6854        fn id(&self) -> String {
6855            self.id.to_string()
6856        }
6857
6858        fn capabilities(&self) -> InferenceCapabilities {
6859            InferenceCapabilities::coding_agent_default()
6860        }
6861
6862        async fn list_models(
6863            &self,
6864            _ctx: InferenceProviderContext<'_>,
6865        ) -> anyhow::Result<Vec<ModelDescriptor>> {
6866            Ok(self.models.clone())
6867        }
6868
6869        async fn stream_turn(
6870            &self,
6871            _ctx: InferenceTurnContext<'_>,
6872            request: AgentInferenceRequest,
6873        ) -> anyhow::Result<InferenceEventStream> {
6874            self.requests.lock().unwrap().push(request);
6875            Ok(Box::pin(stream::iter(vec![
6876                Ok(InferenceEvent::MessageDelta(MessageDelta {
6877                    text: "routed".to_string(),
6878                    phase: None,
6879                })),
6880                Ok(InferenceEvent::Completed(CompletionMetadata {
6881                    stop_reason: Some("stop".to_string()),
6882                    provider_response_id: None,
6883                })),
6884            ])))
6885        }
6886    }
6887
6888    struct StaticRouter {
6889        id: &'static str,
6890        decision: InferenceRoutingDecision,
6891        contexts: Arc<StdMutex<Vec<InferenceRoutingContext>>>,
6892    }
6893
6894    #[async_trait::async_trait]
6895    impl InferenceRouter for StaticRouter {
6896        fn id(&self) -> String {
6897            self.id.to_string()
6898        }
6899
6900        async fn route(
6901            &self,
6902            context: InferenceRoutingContext,
6903        ) -> anyhow::Result<InferenceRoutingDecision> {
6904            self.contexts.lock().unwrap().push(context);
6905            Ok(self.decision.clone())
6906        }
6907    }
6908
6909    fn routing_test_model(id: &str, supported_reasoning: &[&str]) -> ModelDescriptor {
6910        ModelDescriptor {
6911            id: id.to_string(),
6912            name: id.to_string(),
6913            context_window: Some(128_000),
6914            default_reasoning: supported_reasoning
6915                .first()
6916                .map(|effort| (*effort).to_string()),
6917            supported_reasoning: supported_reasoning
6918                .iter()
6919                .map(|effort| ReasoningEffortDescriptor {
6920                    effort: (*effort).to_string(),
6921                    description: format!("{effort} reasoning"),
6922                })
6923                .collect(),
6924        }
6925    }
6926
6927    struct TaskLedgerCompletionGateEngine {
6928        calls: StdMutex<u32>,
6929        requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
6930    }
6931
6932    #[async_trait::async_trait]
6933    impl InferenceEngine for TaskLedgerCompletionGateEngine {
6934        fn id(&self) -> String {
6935            roder_api::catalog::PROVIDER_MOCK.to_string()
6936        }
6937
6938        fn capabilities(&self) -> InferenceCapabilities {
6939            InferenceCapabilities::coding_agent_default()
6940        }
6941
6942        async fn list_models(
6943            &self,
6944            _ctx: InferenceProviderContext<'_>,
6945        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
6946            Ok(roder_api::catalog::models_for_provider(
6947                roder_api::catalog::PROVIDER_MOCK,
6948                true,
6949            ))
6950        }
6951
6952        async fn stream_turn(
6953            &self,
6954            _ctx: InferenceTurnContext<'_>,
6955            request: AgentInferenceRequest,
6956        ) -> anyhow::Result<InferenceEventStream> {
6957            self.requests.lock().unwrap().push(request);
6958            let mut calls = self.calls.lock().unwrap();
6959            *calls += 1;
6960            let events = match *calls {
6961                1 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
6962                    id: "ledger-open".to_string(),
6963                    name: TASK_LEDGER_TOOL_NAME.to_string(),
6964                    arguments: serde_json::json!({
6965                        "tasks": [
6966                            {
6967                                "id": "inspect",
6968                                "content": "Inspect local assets",
6969                                "status": "completed",
6970                                "evidence": "listed workspace"
6971                            },
6972                            {
6973                                "id": "write",
6974                                "content": "Write /app/result.txt",
6975                                "status": "pending"
6976                            }
6977                        ],
6978                        "requireCompletionEvidence": true
6979                    })
6980                    .to_string(),
6981                }))],
6982                3 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
6983                    id: "ledger-complete".to_string(),
6984                    name: TASK_LEDGER_TOOL_NAME.to_string(),
6985                    arguments: serde_json::json!({
6986                        "tasks": [
6987                            {
6988                                "id": "inspect",
6989                                "content": "Inspect local assets",
6990                                "status": "completed",
6991                                "evidence": "listed workspace"
6992                            },
6993                            {
6994                                "id": "write",
6995                                "content": "Write /app/result.txt",
6996                                "status": "completed",
6997                                "evidence": "wrote answer"
6998                            }
6999                        ],
7000                        "requireCompletionEvidence": true
7001                    })
7002                    .to_string(),
7003                }))],
7004                _ => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
7005                    text: "final".to_string(),
7006                    phase: None,
7007                }))],
7008            };
7009            Ok(Box::pin(stream::iter(events.into_iter().chain(
7010                std::iter::once(Ok(InferenceEvent::Completed(CompletionMetadata {
7011                    stop_reason: Some("stop".to_string()),
7012                    provider_response_id: None,
7013                }))),
7014            ))))
7015        }
7016    }
7017
7018    struct VerificationGateEngine {
7019        calls: StdMutex<u32>,
7020    }
7021
7022    #[async_trait::async_trait]
7023    impl InferenceEngine for VerificationGateEngine {
7024        fn id(&self) -> String {
7025            roder_api::catalog::PROVIDER_MOCK.to_string()
7026        }
7027
7028        fn capabilities(&self) -> InferenceCapabilities {
7029            InferenceCapabilities::coding_agent_default()
7030        }
7031
7032        async fn list_models(
7033            &self,
7034            _ctx: InferenceProviderContext<'_>,
7035        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7036            Ok(roder_api::catalog::models_for_provider(
7037                roder_api::catalog::PROVIDER_MOCK,
7038                true,
7039            ))
7040        }
7041
7042        async fn stream_turn(
7043            &self,
7044            _ctx: InferenceTurnContext<'_>,
7045            request: AgentInferenceRequest,
7046        ) -> anyhow::Result<InferenceEventStream> {
7047            let mut calls = self.calls.lock().unwrap();
7048            *calls += 1;
7049            let events = match *calls {
7050                1 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7051                    id: "write-1".to_string(),
7052                    name: "write_file".to_string(),
7053                    arguments: serde_json::json!({
7054                        "path": "src/lib.rs",
7055                        "content": "pub fn answer() -> u8 { 42 }\n"
7056                    })
7057                    .to_string(),
7058                }))],
7059                2 => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
7060                    text: "done too early".to_string(),
7061                    phase: None,
7062                }))],
7063                3 if request.transcript.iter().any(|item| {
7064                    matches!(
7065                        item,
7066                        TranscriptItem::UserMessage(message)
7067                            if message.text.contains("Verification gate blocked final completion")
7068                    )
7069                }) =>
7070                {
7071                    vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7072                        id: "verify-1".to_string(),
7073                        name: crate::verification_gate::VERIFICATION_TOOL_NAME.to_string(),
7074                        arguments: serde_json::json!({
7075                            "originalTask": "write code",
7076                            "changedFiles": ["src/lib.rs"],
7077                            "toolEvidence": ["write_file wrote src/lib.rs"],
7078                            "testsRun": ["cargo test -p roder-core verification_gate"],
7079                            "openGaps": [],
7080                            "status": "completed"
7081                        })
7082                        .to_string(),
7083                    }))]
7084                }
7085                _ => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
7086                    text: "verified final".to_string(),
7087                    phase: None,
7088                }))],
7089            };
7090            Ok(Box::pin(stream::iter(events.into_iter().chain(
7091                std::iter::once(Ok(InferenceEvent::Completed(CompletionMetadata {
7092                    stop_reason: Some("stop".to_string()),
7093                    provider_response_id: None,
7094                }))),
7095            ))))
7096        }
7097    }
7098
7099    struct SpeedPolicyEngine {
7100        calls: StdMutex<u32>,
7101        requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
7102    }
7103
7104    #[async_trait::async_trait]
7105    impl InferenceEngine for SpeedPolicyEngine {
7106        fn id(&self) -> String {
7107            roder_api::catalog::PROVIDER_MOCK.to_string()
7108        }
7109
7110        fn capabilities(&self) -> InferenceCapabilities {
7111            InferenceCapabilities::coding_agent_default()
7112        }
7113
7114        async fn list_models(
7115            &self,
7116            _ctx: InferenceProviderContext<'_>,
7117        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7118            Ok(roder_api::catalog::models_for_provider(
7119                roder_api::catalog::PROVIDER_MOCK,
7120                true,
7121            ))
7122        }
7123
7124        async fn stream_turn(
7125            &self,
7126            _ctx: InferenceTurnContext<'_>,
7127            request: AgentInferenceRequest,
7128        ) -> anyhow::Result<InferenceEventStream> {
7129            self.requests.lock().unwrap().push(request.clone());
7130            let mut calls = self.calls.lock().unwrap();
7131            *calls += 1;
7132            let events = match *calls {
7133                1 => vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7134                    id: "write-1".to_string(),
7135                    name: "write_file".to_string(),
7136                    arguments: serde_json::json!({
7137                        "path": "src/lib.rs",
7138                        "content": "pub fn answer() -> u8 { 42 }\n"
7139                    })
7140                    .to_string(),
7141                }))],
7142                3 if request.transcript.iter().any(|item| {
7143                    matches!(
7144                        item,
7145                        TranscriptItem::UserMessage(message)
7146                            if message.text.contains("Verification gate blocked final completion")
7147                    )
7148                }) =>
7149                {
7150                    vec![Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7151                        id: "verify-1".to_string(),
7152                        name: crate::verification_gate::VERIFICATION_TOOL_NAME.to_string(),
7153                        arguments: serde_json::json!({
7154                            "originalTask": "write code",
7155                            "changedFiles": ["src/lib.rs"],
7156                            "toolEvidence": ["write_file wrote src/lib.rs"],
7157                            "testsRun": ["cargo test -p roder-core speed_policy"],
7158                            "openGaps": [],
7159                            "status": "completed"
7160                        })
7161                        .to_string(),
7162                    }))]
7163                }
7164                _ => vec![Ok(InferenceEvent::MessageDelta(MessageDelta {
7165                    text: "done".to_string(),
7166                    phase: None,
7167                }))],
7168            };
7169            Ok(Box::pin(stream::iter(events.into_iter().chain(
7170                std::iter::once(Ok(InferenceEvent::Completed(CompletionMetadata {
7171                    stop_reason: Some("stop".to_string()),
7172                    provider_response_id: None,
7173                }))),
7174            ))))
7175        }
7176    }
7177
7178    struct SwitchCaptureEngine {
7179        requests: Arc<StdMutex<Vec<AgentInferenceRequest>>>,
7180    }
7181
7182    #[async_trait::async_trait]
7183    impl InferenceEngine for SwitchCaptureEngine {
7184        fn id(&self) -> String {
7185            roder_api::catalog::PROVIDER_MOCK.to_string()
7186        }
7187
7188        fn capabilities(&self) -> InferenceCapabilities {
7189            InferenceCapabilities::coding_agent_default()
7190        }
7191
7192        async fn list_models(
7193            &self,
7194            _ctx: InferenceProviderContext<'_>,
7195        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7196            Ok(roder_api::catalog::models_for_provider(
7197                roder_api::catalog::PROVIDER_MOCK,
7198                true,
7199            ))
7200        }
7201
7202        async fn stream_turn(
7203            &self,
7204            _ctx: InferenceTurnContext<'_>,
7205            request: AgentInferenceRequest,
7206        ) -> anyhow::Result<InferenceEventStream> {
7207            self.requests.lock().unwrap().push(request);
7208            Ok(Box::pin(stream::iter(vec![
7209                Ok(InferenceEvent::MessageDelta(MessageDelta {
7210                    text: "done".to_string(),
7211                    phase: None,
7212                })),
7213                Ok(InferenceEvent::Completed(CompletionMetadata {
7214                    stop_reason: Some("stop".to_string()),
7215                    provider_response_id: None,
7216                })),
7217            ])))
7218        }
7219    }
7220
7221    struct DeadlineEngine;
7222
7223    #[async_trait::async_trait]
7224    impl InferenceEngine for DeadlineEngine {
7225        fn id(&self) -> String {
7226            roder_api::catalog::PROVIDER_MOCK.to_string()
7227        }
7228
7229        fn capabilities(&self) -> InferenceCapabilities {
7230            InferenceCapabilities::coding_agent_default()
7231        }
7232
7233        async fn list_models(
7234            &self,
7235            _ctx: InferenceProviderContext<'_>,
7236        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7237            Ok(Vec::new())
7238        }
7239
7240        async fn stream_turn(
7241            &self,
7242            _ctx: InferenceTurnContext<'_>,
7243            _request: AgentInferenceRequest,
7244        ) -> anyhow::Result<InferenceEventStream> {
7245            Ok(Box::pin(stream::once(async {
7246                tokio::time::sleep(std::time::Duration::from_secs(60)).await;
7247                Ok(InferenceEvent::MessageDelta(MessageDelta {
7248                    text: "too late".to_string(),
7249                    phase: None,
7250                }))
7251            })))
7252        }
7253    }
7254
7255    struct WriteFileContributor;
7256
7257    impl ToolContributor for WriteFileContributor {
7258        fn id(&self) -> String {
7259            "test-write".to_string()
7260        }
7261
7262        fn contribute(&self, registry: &mut ToolRegistry) -> anyhow::Result<()> {
7263            registry.register(Arc::new(WriteFileTool))
7264        }
7265    }
7266
7267    struct WriteFileTool;
7268
7269    #[async_trait::async_trait]
7270    impl ToolExecutor for WriteFileTool {
7271        fn spec(&self) -> ToolSpec {
7272            ToolSpec {
7273                name: "write_file".to_string(),
7274                description: "Write a test file.".to_string(),
7275                parameters: serde_json::json!({
7276                    "type": "object",
7277                    "properties": {
7278                        "path": { "type": "string" },
7279                        "content": { "type": "string" }
7280                    },
7281                    "required": ["path", "content"],
7282                    "additionalProperties": false
7283                }),
7284            }
7285        }
7286
7287        async fn execute(
7288            &self,
7289            _ctx: ToolExecutionContext,
7290            call: ToolCall,
7291        ) -> anyhow::Result<ToolResult> {
7292            let path = call
7293                .arguments
7294                .get("path")
7295                .and_then(serde_json::Value::as_str)
7296                .unwrap_or("src/lib.rs");
7297            Ok(ToolResult {
7298                id: call.id,
7299                name: call.name,
7300                text: format!("wrote {path}"),
7301                data: serde_json::json!({ "path": path }),
7302                is_error: false,
7303            })
7304        }
7305    }
7306
7307    struct ProfileToolContributor;
7308
7309    impl ToolContributor for ProfileToolContributor {
7310        fn id(&self) -> String {
7311            "profile-tools".to_string()
7312        }
7313
7314        fn contribute(&self, registry: &mut ToolRegistry) -> anyhow::Result<()> {
7315            for name in ["apply_patch", "edit", "multi_edit", "write_file"] {
7316                registry.register(Arc::new(ProfileTool {
7317                    name: name.to_string(),
7318                }))?;
7319            }
7320            Ok(())
7321        }
7322    }
7323
7324    struct ProfileTool {
7325        name: String,
7326    }
7327
7328    #[async_trait::async_trait]
7329    impl ToolExecutor for ProfileTool {
7330        fn spec(&self) -> ToolSpec {
7331            ToolSpec {
7332                name: self.name.clone(),
7333                description: format!("{} test tool", self.name),
7334                parameters: serde_json::json!({
7335                    "type": "object",
7336                    "properties": {
7337                        "path": { "type": "string" },
7338                        "content": { "type": "string" }
7339                    },
7340                    "required": ["path", "content"],
7341                    "additionalProperties": false
7342                }),
7343            }
7344        }
7345
7346        async fn execute(
7347            &self,
7348            _ctx: ToolExecutionContext,
7349            call: ToolCall,
7350        ) -> anyhow::Result<ToolResult> {
7351            Ok(ToolResult {
7352                id: call.id,
7353                name: call.name,
7354                text: "ok".to_string(),
7355                data: serde_json::json!({}),
7356                is_error: false,
7357            })
7358        }
7359    }
7360
7361    fn test_model_profile(model: &str) -> ModelHarnessProfile {
7362        ModelHarnessProfile {
7363            model: model.to_string(),
7364            provider: roder_api::catalog::PROVIDER_OPENAI.to_string(),
7365            provider_family: ProviderFamily::OpenAi,
7366            edit_tool: Some(EDIT_TOOL_EDIT.to_string()),
7367            schema_policy: ModelSchemaPolicy::StandardRequiredFirst,
7368            instruction_overlay: ModelInstructionOverlay::IntuitiveContext,
7369            reasoning: ModelProfileReasoning {
7370                orientation: Some(REASONING_LOW.to_string()),
7371                execution: Some(REASONING_LOW.to_string()),
7372                verification: Some(REASONING_LOW.to_string()),
7373                recovery: Some(REASONING_LOW.to_string()),
7374            },
7375            parallel_tool_calls: Some(false),
7376            auto_compact_token_limit: Some(123_000),
7377        }
7378    }
7379
7380    async fn captured_profile_request(cfg: RuntimeConfig) -> AgentInferenceRequest {
7381        let captured = Arc::new(StdMutex::new(None));
7382        let mut builder = ExtensionRegistryBuilder::new();
7383        builder.inference_engine(Arc::new(CapturingEngine {
7384            request: captured.clone(),
7385        }));
7386        builder.tool_contributor(Arc::new(ProfileToolContributor));
7387        let runtime = Arc::new(Runtime::new(builder.build().unwrap(), cfg).unwrap());
7388        let mut rx = runtime.subscribe_events();
7389        let turn_id = runtime
7390            .start_turn(StartTurnRequest {
7391                thread_id: "thread-model-profile".to_string(),
7392                message: "use profile knobs".to_string(),
7393                images: Vec::new(),
7394                provider_override: None,
7395                model_override: None,
7396                reasoning_override: None,
7397                workspace: test_workspace(),
7398                instructions: InstructionBundle {
7399                    system: None,
7400                    developer: Some("base developer".to_string()),
7401                    developer_context: None,
7402                },
7403                developer_context: None,
7404                task_ledger_required: false,
7405                service_tier_override: None,
7406            })
7407            .await
7408            .unwrap();
7409
7410        tokio::time::timeout(std::time::Duration::from_secs(5), async {
7411            loop {
7412                let envelope = rx.recv().await.unwrap();
7413                if envelope.turn_id.as_deref() != Some(&turn_id) {
7414                    continue;
7415                }
7416                match envelope.event {
7417                    RoderEvent::TurnCompleted(_) => break,
7418                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7419                    _ => {}
7420                }
7421            }
7422        })
7423        .await
7424        .unwrap();
7425
7426        captured.lock().unwrap().clone().unwrap()
7427    }
7428
7429    struct ToolThenStopEngine {
7430        calls: StdMutex<u32>,
7431    }
7432
7433    #[async_trait::async_trait]
7434    impl InferenceEngine for ToolThenStopEngine {
7435        fn id(&self) -> String {
7436            roder_api::catalog::PROVIDER_MOCK.to_string()
7437        }
7438
7439        fn capabilities(&self) -> InferenceCapabilities {
7440            InferenceCapabilities::coding_agent_default()
7441        }
7442
7443        async fn list_models(
7444            &self,
7445            _ctx: InferenceProviderContext<'_>,
7446        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
7447            Ok(roder_api::catalog::models_for_provider(
7448                roder_api::catalog::PROVIDER_MOCK,
7449                true,
7450            ))
7451        }
7452
7453        async fn stream_turn(
7454            &self,
7455            _ctx: InferenceTurnContext<'_>,
7456            _request: AgentInferenceRequest,
7457        ) -> anyhow::Result<InferenceEventStream> {
7458            let mut calls = self.calls.lock().unwrap();
7459            *calls += 1;
7460            let events = match *calls {
7461                1 => vec![
7462                    Ok(InferenceEvent::ToolCallCompleted(ToolCallCompleted {
7463                        id: "write-1".to_string(),
7464                        name: "write_file".to_string(),
7465                        arguments: serde_json::json!({
7466                            "path": "src/lib.rs",
7467                            "content": "pub fn answer() -> u8 { 42 }\n"
7468                        })
7469                        .to_string(),
7470                    })),
7471                    Ok(InferenceEvent::Completed(CompletionMetadata {
7472                        stop_reason: Some("tool_use".to_string()),
7473                        provider_response_id: None,
7474                    })),
7475                ],
7476                _ => vec![
7477                    Ok(InferenceEvent::MessageDelta(MessageDelta {
7478                        text: "final".to_string(),
7479                        phase: None,
7480                    })),
7481                    Ok(InferenceEvent::Completed(CompletionMetadata {
7482                        stop_reason: Some("end_turn".to_string()),
7483                        provider_response_id: None,
7484                    })),
7485                ],
7486            };
7487            Ok(Box::pin(stream::iter(events)))
7488        }
7489    }
7490
7491    #[tokio::test]
7492    async fn turn_completed_reports_terminal_step_finish_reason() {
7493        let mut builder = ExtensionRegistryBuilder::new();
7494        builder.inference_engine(Arc::new(ToolThenStopEngine {
7495            calls: StdMutex::new(0),
7496        }));
7497        builder.tool_contributor(Arc::new(WriteFileContributor));
7498        let runtime = Arc::new(
7499            Runtime::new(
7500                builder.build().unwrap(),
7501                RuntimeConfig {
7502                    policy_mode: PolicyMode::Bypass,
7503                    agent_swarm_mode: false,
7504                    ultra_mode: false,
7505                    ..RuntimeConfig::default()
7506                },
7507            )
7508            .unwrap(),
7509        );
7510        let mut rx = runtime.subscribe_events();
7511        let turn_id = runtime
7512            .start_turn(StartTurnRequest {
7513                thread_id: "thread-finish-reason".to_string(),
7514                message: "write then finish".to_string(),
7515                images: Vec::new(),
7516                provider_override: None,
7517                model_override: None,
7518                reasoning_override: None,
7519                workspace: test_workspace(),
7520                instructions: InstructionBundle {
7521                    system: None,
7522                    developer: None,
7523                    developer_context: None,
7524                },
7525                developer_context: None,
7526                task_ledger_required: false,
7527                service_tier_override: None,
7528            })
7529            .await
7530            .unwrap();
7531
7532        let completed = tokio::time::timeout(std::time::Duration::from_secs(5), async {
7533            loop {
7534                let envelope = rx.recv().await.unwrap();
7535                if envelope.turn_id.as_deref() != Some(&turn_id) {
7536                    continue;
7537                }
7538                match envelope.event {
7539                    RoderEvent::TurnCompleted(event) => break event,
7540                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7541                    _ => {}
7542                }
7543            }
7544        })
7545        .await
7546        .unwrap();
7547
7548        // The mid-turn tool_use stop reason must not leak; the terminal
7549        // end_turn step decides the turn's finish reason.
7550        assert_eq!(completed.finish_reason.as_deref(), Some("stop"));
7551    }
7552
7553    #[tokio::test]
7554    async fn inference_router_selection_changes_request_model_and_records_event() {
7555        let default_requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7556        let routed_requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7557        let contexts = Arc::new(StdMutex::new(Vec::<InferenceRoutingContext>::new()));
7558        let selected = ModelSelection {
7559            provider: "routed-provider".to_string(),
7560            model: "routed-model".to_string(),
7561        };
7562        let default = ModelSelection {
7563            provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7564            model: "mock".to_string(),
7565        };
7566        let decision = InferenceRoutingDecision {
7567            reasoning: Some(ReasoningConfig {
7568                enabled: true,
7569                level: Some(REASONING_LOW.to_string()),
7570            }),
7571            confidence: Some(0.91),
7572            baseline: Some(default.clone()),
7573            matched_signals: vec![roder_api::inference_routing::InferenceRoutingSignal::new(
7574                "intent", "routine",
7575            )],
7576            ..InferenceRoutingDecision::selected("test-router", selected.clone(), "routine request")
7577        };
7578
7579        let mut builder = ExtensionRegistryBuilder::new();
7580        builder.inference_engine(Arc::new(RoutingCaptureEngine {
7581            id: roder_api::catalog::PROVIDER_MOCK,
7582            models: vec![routing_test_model("mock", &[REASONING_LOW])],
7583            requests: default_requests.clone(),
7584        }));
7585        builder.inference_engine(Arc::new(RoutingCaptureEngine {
7586            id: "routed-provider",
7587            models: vec![routing_test_model(
7588                "routed-model",
7589                &[REASONING_LOW, REASONING_MEDIUM],
7590            )],
7591            requests: routed_requests.clone(),
7592        }));
7593        builder.inference_router(Arc::new(StaticRouter {
7594            id: "test-router",
7595            decision,
7596            contexts: contexts.clone(),
7597        }));
7598        let thread_root =
7599            std::env::temp_dir().join(format!("roder-routing-auto-{}", uuid::Uuid::new_v4()));
7600        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
7601            base_path: thread_root.clone(),
7602        }));
7603        let runtime = Arc::new(
7604            Runtime::new(
7605                builder.build().unwrap(),
7606                RuntimeConfig {
7607                    default_provider: default.provider.clone(),
7608                    default_model: default.model.clone(),
7609                    ..RuntimeConfig::default()
7610                },
7611            )
7612            .unwrap(),
7613        );
7614        let thread_id = runtime
7615            .create_thread_with(CreateThreadRequest {
7616                title: Some("Routing auto".to_string()),
7617                workspace: test_workspace(),
7618                workspace_id: None,
7619                root_id: None,
7620                provider: Some(default.provider.clone()),
7621                model: Some(default.model.clone()),
7622                tool_allowlist: Vec::new(),
7623                service_tier: None,
7624                developer_instructions: None,
7625                external_tools: Vec::new(),
7626                selection_mode: Some(ModelSelectionMode::auto(
7627                    "test-router:coding",
7628                    "test-router",
7629                    "Auto: Coding",
7630                    default.clone(),
7631                    Some("coding".to_string()),
7632                    None,
7633                )),
7634                runner: None,
7635            })
7636            .await
7637            .unwrap()
7638            .thread_id;
7639        let mut rx = runtime.subscribe_events();
7640        let turn_id = runtime
7641            .start_turn(StartTurnRequest {
7642                thread_id: thread_id.clone(),
7643                message: "small cleanup".to_string(),
7644                images: Vec::new(),
7645                provider_override: None,
7646                model_override: None,
7647                reasoning_override: None,
7648                workspace: test_workspace(),
7649                instructions: InstructionBundle::default(),
7650                developer_context: None,
7651                task_ledger_required: false,
7652                service_tier_override: None,
7653            })
7654            .await
7655            .unwrap();
7656
7657        let mut routing_event = None;
7658        let mut inference_started = None;
7659        tokio::time::timeout(std::time::Duration::from_secs(5), async {
7660            loop {
7661                let envelope = rx.recv().await.unwrap();
7662                if envelope.turn_id.as_deref() != Some(&turn_id) {
7663                    continue;
7664                }
7665                match envelope.event {
7666                    RoderEvent::InferenceRoutingDecision(event) => {
7667                        routing_event = Some(event);
7668                    }
7669                    RoderEvent::InferenceStarted(event) => {
7670                        inference_started = Some(event);
7671                    }
7672                    RoderEvent::TurnCompleted(_) => break,
7673                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7674                    _ => {}
7675                }
7676            }
7677        })
7678        .await
7679        .unwrap();
7680
7681        assert!(default_requests.lock().unwrap().is_empty());
7682        let routed_requests = routed_requests.lock().unwrap();
7683        assert_eq!(routed_requests.len(), 1);
7684        assert_eq!(routed_requests[0].model, selected);
7685        assert_eq!(
7686            routed_requests[0].reasoning.level.as_deref(),
7687            Some(REASONING_LOW)
7688        );
7689        assert_eq!(
7690            routed_requests[0].metadata["inferenceRouting"]["outcome"],
7691            "selected"
7692        );
7693
7694        let routing_event = routing_event.expect("routing decision event");
7695        assert_eq!(routing_event.default_selection, default);
7696        assert_eq!(routing_event.selected_selection, selected);
7697        assert_eq!(
7698            routing_event.decision.outcome,
7699            InferenceRoutingOutcome::Selected
7700        );
7701        assert_eq!(
7702            inference_started.expect("inference started event").model,
7703            selected
7704        );
7705
7706        let contexts = contexts.lock().unwrap();
7707        assert_eq!(contexts.len(), 1);
7708        assert_eq!(contexts[0].default_selection, default);
7709        assert_eq!(contexts[0].candidates.len(), 2);
7710        assert!(
7711            contexts[0]
7712                .signals
7713                .iter()
7714                .any(|signal| signal.key == "profile" && signal.value == "coding")
7715        );
7716        let _ = std::fs::remove_dir_all(thread_root);
7717    }
7718
7719    #[tokio::test]
7720    async fn inference_router_is_bypassed_for_explicit_selection() {
7721        let requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7722        let contexts = Arc::new(StdMutex::new(Vec::<InferenceRoutingContext>::new()));
7723        let selected = ModelSelection {
7724            provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7725            model: "mock".to_string(),
7726        };
7727
7728        let mut builder = ExtensionRegistryBuilder::new();
7729        builder.inference_engine(Arc::new(RoutingCaptureEngine {
7730            id: roder_api::catalog::PROVIDER_MOCK,
7731            models: vec![routing_test_model("mock", &[REASONING_LOW])],
7732            requests: requests.clone(),
7733        }));
7734        builder.inference_router(Arc::new(StaticRouter {
7735            id: "test-router",
7736            decision: InferenceRoutingDecision::selected(
7737                "test-router",
7738                ModelSelection {
7739                    provider: "missing".to_string(),
7740                    model: "missing".to_string(),
7741                },
7742                "would route if called",
7743            ),
7744            contexts: contexts.clone(),
7745        }));
7746        let thread_root =
7747            std::env::temp_dir().join(format!("roder-routing-explicit-{}", uuid::Uuid::new_v4()));
7748        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
7749            base_path: thread_root.clone(),
7750        }));
7751        let runtime = Arc::new(
7752            Runtime::new(
7753                builder.build().unwrap(),
7754                RuntimeConfig {
7755                    default_provider: selected.provider.clone(),
7756                    default_model: selected.model.clone(),
7757                    inference_router: RuntimeInferenceRouterConfig {
7758                        enabled: true,
7759                        router_id: Some("test-router".to_string()),
7760                    },
7761                    ..RuntimeConfig::default()
7762                },
7763            )
7764            .unwrap(),
7765        );
7766        let thread_id = runtime
7767            .create_thread_with(CreateThreadRequest {
7768                title: Some("Routing explicit".to_string()),
7769                workspace: test_workspace(),
7770                workspace_id: None,
7771                root_id: None,
7772                provider: Some(selected.provider.clone()),
7773                model: Some(selected.model.clone()),
7774                tool_allowlist: Vec::new(),
7775                service_tier: None,
7776                developer_instructions: None,
7777                external_tools: Vec::new(),
7778                selection_mode: Some(ModelSelectionMode::auto(
7779                    "test-router:default",
7780                    "test-router",
7781                    "Auto",
7782                    selected.clone(),
7783                    None,
7784                    None,
7785                )),
7786                runner: None,
7787            })
7788            .await
7789            .unwrap()
7790            .thread_id;
7791        let mut rx = runtime.subscribe_events();
7792        let turn_id = runtime
7793            .start_turn(StartTurnRequest {
7794                thread_id,
7795                message: "use explicit selection".to_string(),
7796                images: Vec::new(),
7797                provider_override: Some(selected.provider.clone()),
7798                model_override: Some(selected.model.clone()),
7799                reasoning_override: None,
7800                workspace: test_workspace(),
7801                instructions: InstructionBundle::default(),
7802                developer_context: None,
7803                task_ledger_required: false,
7804                service_tier_override: None,
7805            })
7806            .await
7807            .unwrap();
7808
7809        let mut saw_routing_event = false;
7810        tokio::time::timeout(std::time::Duration::from_secs(5), async {
7811            loop {
7812                let envelope = rx.recv().await.unwrap();
7813                if envelope.turn_id.as_deref() != Some(&turn_id) {
7814                    continue;
7815                }
7816                match envelope.event {
7817                    RoderEvent::InferenceRoutingDecision(_) => {
7818                        saw_routing_event = true;
7819                    }
7820                    RoderEvent::TurnCompleted(_) => break,
7821                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7822                    _ => {}
7823                }
7824            }
7825        })
7826        .await
7827        .unwrap();
7828
7829        assert!(!saw_routing_event);
7830        assert!(contexts.lock().unwrap().is_empty());
7831        let requests = requests.lock().unwrap();
7832        assert_eq!(requests.len(), 1);
7833        assert_eq!(requests[0].model, selected);
7834        let _ = std::fs::remove_dir_all(thread_root);
7835    }
7836
7837    #[tokio::test]
7838    async fn inference_router_is_bypassed_for_manual_selection_mode() {
7839        let requests = Arc::new(StdMutex::new(Vec::<AgentInferenceRequest>::new()));
7840        let contexts = Arc::new(StdMutex::new(Vec::<InferenceRoutingContext>::new()));
7841        let selected = ModelSelection {
7842            provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7843            model: "mock".to_string(),
7844        };
7845
7846        let mut builder = ExtensionRegistryBuilder::new();
7847        builder.inference_engine(Arc::new(RoutingCaptureEngine {
7848            id: roder_api::catalog::PROVIDER_MOCK,
7849            models: vec![routing_test_model("mock", &[REASONING_LOW])],
7850            requests: requests.clone(),
7851        }));
7852        builder.inference_router(Arc::new(StaticRouter {
7853            id: "test-router",
7854            decision: InferenceRoutingDecision::selected(
7855                "test-router",
7856                ModelSelection {
7857                    provider: "missing".to_string(),
7858                    model: "missing".to_string(),
7859                },
7860                "would route if called",
7861            ),
7862            contexts: contexts.clone(),
7863        }));
7864        let thread_root =
7865            std::env::temp_dir().join(format!("roder-routing-manual-{}", uuid::Uuid::new_v4()));
7866        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
7867            base_path: thread_root.clone(),
7868        }));
7869        let runtime = Arc::new(
7870            Runtime::new(
7871                builder.build().unwrap(),
7872                RuntimeConfig {
7873                    default_provider: selected.provider.clone(),
7874                    default_model: selected.model.clone(),
7875                    inference_router: RuntimeInferenceRouterConfig {
7876                        enabled: true,
7877                        router_id: Some("test-router".to_string()),
7878                    },
7879                    ..RuntimeConfig::default()
7880                },
7881            )
7882            .unwrap(),
7883        );
7884        let thread_id = runtime
7885            .create_thread_with(CreateThreadRequest {
7886                title: Some("Routing manual".to_string()),
7887                workspace: test_workspace(),
7888                workspace_id: None,
7889                root_id: None,
7890                provider: Some(selected.provider.clone()),
7891                model: Some(selected.model.clone()),
7892                tool_allowlist: Vec::new(),
7893                service_tier: None,
7894                developer_instructions: None,
7895                external_tools: Vec::new(),
7896                selection_mode: Some(ModelSelectionMode::manual(
7897                    selected.provider.clone(),
7898                    selected.model.clone(),
7899                    None,
7900                )),
7901                runner: None,
7902            })
7903            .await
7904            .unwrap()
7905            .thread_id;
7906        let mut rx = runtime.subscribe_events();
7907        let turn_id = runtime
7908            .start_turn(StartTurnRequest {
7909                thread_id,
7910                message: "use selected manual model".to_string(),
7911                images: Vec::new(),
7912                provider_override: None,
7913                model_override: None,
7914                reasoning_override: None,
7915                workspace: test_workspace(),
7916                instructions: InstructionBundle::default(),
7917                developer_context: None,
7918                task_ledger_required: false,
7919                service_tier_override: None,
7920            })
7921            .await
7922            .unwrap();
7923
7924        let mut saw_routing_event = false;
7925        tokio::time::timeout(std::time::Duration::from_secs(5), async {
7926            loop {
7927                let envelope = rx.recv().await.unwrap();
7928                if envelope.turn_id.as_deref() != Some(&turn_id) {
7929                    continue;
7930                }
7931                match envelope.event {
7932                    RoderEvent::InferenceRoutingDecision(_) => {
7933                        saw_routing_event = true;
7934                    }
7935                    RoderEvent::TurnCompleted(_) => break,
7936                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
7937                    _ => {}
7938                }
7939            }
7940        })
7941        .await
7942        .unwrap();
7943
7944        assert!(!saw_routing_event);
7945        assert!(contexts.lock().unwrap().is_empty());
7946        let requests = requests.lock().unwrap();
7947        assert_eq!(requests.len(), 1);
7948        assert_eq!(requests[0].model, selected);
7949        let _ = std::fs::remove_dir_all(thread_root);
7950    }
7951
7952    #[test]
7953    fn enabled_inference_router_requires_registered_router() {
7954        let mut builder = ExtensionRegistryBuilder::new();
7955        builder.inference_engine(Arc::new(FakeInferenceEngine));
7956
7957        let err = match Runtime::new(
7958            builder.build().unwrap(),
7959            RuntimeConfig {
7960                inference_router: RuntimeInferenceRouterConfig {
7961                    enabled: true,
7962                    router_id: Some("missing-router".to_string()),
7963                },
7964                ..RuntimeConfig::default()
7965            },
7966        ) {
7967            Ok(_) => panic!("runtime should reject unknown inference router"),
7968            Err(err) => err,
7969        };
7970
7971        assert!(
7972            err.to_string()
7973                .contains("inference router \"missing-router\" is not registered")
7974        );
7975    }
7976
7977    #[tokio::test]
7978    async fn model_profile_routes_request_knobs_to_next_inference() {
7979        let request = captured_profile_request(RuntimeConfig {
7980            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
7981            default_model: "gpt-5.5".to_string(),
7982            model_profiles: std::collections::HashMap::from([(
7983                "gpt-5.5".to_string(),
7984                test_model_profile("gpt-5.5"),
7985            )]),
7986            ..RuntimeConfig::default()
7987        })
7988        .await;
7989
7990        let tool_names = request
7991            .tools
7992            .iter()
7993            .map(|tool| tool.name.as_str())
7994            .collect::<Vec<_>>();
7995        assert!(tool_names.contains(&"apply_patch"));
7996        assert!(tool_names.contains(&"edit"));
7997        assert!(tool_names.contains(&"multi_edit"));
7998        assert!(tool_names.contains(&"write_file"));
7999        assert_eq!(request.reasoning.level.as_deref(), Some(REASONING_LOW));
8000        assert_eq!(request.runtime.parallel_tool_calls, Some(false));
8001        assert_eq!(request.runtime.auto_compact_token_limit, Some(123_000));
8002        assert!(
8003            request
8004                .instructions
8005                .developer
8006                .as_deref()
8007                .unwrap_or_default()
8008                .contains("Use the provided context as the current working set")
8009        );
8010        assert_eq!(
8011            request
8012                .metadata
8013                .pointer("/modelProfile/schemaPolicy")
8014                .and_then(serde_json::Value::as_str),
8015            Some("standard_required_first")
8016        );
8017    }
8018
8019    #[tokio::test]
8020    async fn ultra_reasoning_reaches_transport_state_and_enables_proactive_delegation() {
8021        let request = captured_profile_request(RuntimeConfig {
8022            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8023            default_model: "gpt-5.6-sol".to_string(),
8024            reasoning: Some(REASONING_ULTRA.to_string()),
8025            ..RuntimeConfig::default()
8026        })
8027        .await;
8028
8029        assert_eq!(request.reasoning.level.as_deref(), Some(REASONING_ULTRA));
8030        let developer = request
8031            .instructions
8032            .developer
8033            .as_deref()
8034            .expect("ultra developer instructions");
8035        assert!(developer.contains("Proactive multi-agent delegation is active"));
8036    }
8037
8038    #[tokio::test]
8039    async fn lower_sol_effort_requires_explicit_multi_agent_request() {
8040        let request = captured_profile_request(RuntimeConfig {
8041            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8042            default_model: "gpt-5.6-sol".to_string(),
8043            reasoning: Some(roder_api::catalog::REASONING_MEDIUM.to_string()),
8044            ..RuntimeConfig::default()
8045        })
8046        .await;
8047
8048        let developer = request
8049            .instructions
8050            .developer
8051            .as_deref()
8052            .expect("sol developer instructions");
8053        assert!(developer.contains("Do not spawn sub-agents unless"));
8054        assert!(!developer.contains("Proactive multi-agent delegation is active"));
8055    }
8056
8057    #[tokio::test]
8058    async fn luna_does_not_receive_codex_v2_multi_agent_policy() {
8059        let request = captured_profile_request(RuntimeConfig {
8060            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8061            default_model: "gpt-5.6-luna".to_string(),
8062            reasoning: Some(roder_api::catalog::REASONING_MAX.to_string()),
8063            ..RuntimeConfig::default()
8064        })
8065        .await;
8066
8067        let developer = request
8068            .instructions
8069            .developer
8070            .as_deref()
8071            .unwrap_or_default();
8072        assert!(!developer.contains("Do not spawn sub-agents unless"));
8073        assert!(!developer.contains("Proactive multi-agent delegation is active"));
8074    }
8075
8076    #[tokio::test]
8077    async fn ultra_mode_enables_proactive_multi_agent_for_any_model() {
8078        // Luna has no Ultra effort, but ultra mode still injects proactive policy.
8079        let request = captured_profile_request(RuntimeConfig {
8080            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8081            default_model: "gpt-5.6-luna".to_string(),
8082            reasoning: Some(roder_api::catalog::REASONING_MAX.to_string()),
8083            ultra_mode: true,
8084            ..RuntimeConfig::default()
8085        })
8086        .await;
8087
8088        let developer = request
8089            .instructions
8090            .developer
8091            .as_deref()
8092            .expect("ultra mode developer instructions");
8093        assert!(developer.contains("Proactive multi-agent delegation is active"));
8094        assert!(developer.contains("spawn_agent"));
8095    }
8096
8097    #[tokio::test]
8098    async fn ultra_mode_overrides_explicit_request_only_on_sol() {
8099        // Medium Sol effort is normally explicit-request-only; ultra mode flips
8100        // it to proactive without requiring Ultra reasoning effort.
8101        let request = captured_profile_request(RuntimeConfig {
8102            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8103            default_model: "gpt-5.6-sol".to_string(),
8104            reasoning: Some(roder_api::catalog::REASONING_MEDIUM.to_string()),
8105            ultra_mode: true,
8106            ..RuntimeConfig::default()
8107        })
8108        .await;
8109
8110        let developer = request
8111            .instructions
8112            .developer
8113            .as_deref()
8114            .expect("ultra mode sol developer instructions");
8115        assert!(developer.contains("Proactive multi-agent delegation is active"));
8116        assert!(!developer.contains("Do not spawn sub-agents unless"));
8117    }
8118
8119    #[tokio::test]
8120    async fn turn_developer_context_reaches_inference_and_does_not_persist() {
8121        let captured = Arc::new(StdMutex::new(None));
8122        let mut builder = ExtensionRegistryBuilder::new();
8123        builder.inference_engine(Arc::new(CapturingEngine {
8124            request: captured.clone(),
8125        }));
8126        let runtime = Arc::new(
8127            Runtime::new(
8128                builder.build().unwrap(),
8129                RuntimeConfig {
8130                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8131                    default_model: "gpt-5.5".to_string(),
8132                    ..RuntimeConfig::default()
8133                },
8134            )
8135            .unwrap(),
8136        );
8137
8138        async fn run_turn(runtime: &Arc<Runtime>, developer_context: Option<String>) {
8139            let mut rx = runtime.subscribe_events();
8140            let turn_id = runtime
8141                .start_turn(StartTurnRequest {
8142                    thread_id: "thread-turn-context".to_string(),
8143                    message: "hello".to_string(),
8144                    images: Vec::new(),
8145                    provider_override: None,
8146                    model_override: None,
8147                    reasoning_override: None,
8148                    workspace: test_workspace(),
8149                    instructions: InstructionBundle::default(),
8150                    developer_context,
8151                    task_ledger_required: false,
8152                    service_tier_override: None,
8153                })
8154                .await
8155                .unwrap();
8156            tokio::time::timeout(std::time::Duration::from_secs(5), async {
8157                loop {
8158                    let envelope = rx.recv().await.unwrap();
8159                    if envelope.turn_id.as_deref() != Some(&turn_id) {
8160                        continue;
8161                    }
8162                    match envelope.event {
8163                        RoderEvent::TurnCompleted(_) => break,
8164                        RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8165                        _ => {}
8166                    }
8167                }
8168            })
8169            .await
8170            .unwrap();
8171        }
8172
8173        run_turn(
8174            &runtime,
8175            Some("Connected accounts: example-service.".to_string()),
8176        )
8177        .await;
8178        let request = captured.lock().unwrap().clone().unwrap();
8179        assert_eq!(
8180            request.instructions.developer_context.as_deref(),
8181            Some("Connected accounts: example-service.")
8182        );
8183
8184        // The context is per-turn only: the next turn on the same thread
8185        // without a developerContext must not see the previous one.
8186        run_turn(&runtime, None).await;
8187        let request = captured.lock().unwrap().clone().unwrap();
8188        assert_eq!(request.instructions.developer_context, None);
8189    }
8190
8191    #[tokio::test]
8192    async fn tool_search_overrides_route_to_next_inference_request() {
8193        let request = captured_profile_request(RuntimeConfig {
8194            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8195            default_model: "gpt-5.4".to_string(),
8196            tool_search: ToolSearchConfig {
8197                mode: roder_api::inference::ToolSearchMode::Auto,
8198                max_catalog_items: Some(100),
8199                ..ToolSearchConfig::default()
8200            },
8201            provider_tool_search: std::collections::HashMap::from([(
8202                roder_api::catalog::PROVIDER_MOCK.to_string(),
8203                roder_api::inference::ToolSearchConfigOverlay {
8204                    include_skills: Some(false),
8205                    provider_variant: Some(roder_api::inference::ToolSearchProviderVariant::Regex),
8206                    ..Default::default()
8207                },
8208            )]),
8209            model_tool_search: std::collections::HashMap::from([(
8210                "gpt-5.4".to_string(),
8211                roder_api::inference::ToolSearchConfigOverlay {
8212                    mode: Some(roder_api::inference::ToolSearchMode::ProviderNative),
8213                    max_catalog_items: Some(25),
8214                    provider_variant: Some(roder_api::inference::ToolSearchProviderVariant::Bm25),
8215                    ..Default::default()
8216                },
8217            )]),
8218            ..RuntimeConfig::default()
8219        })
8220        .await;
8221
8222        assert_eq!(
8223            request.runtime.tool_search.mode,
8224            roder_api::inference::ToolSearchMode::ProviderNative
8225        );
8226        assert_eq!(request.runtime.tool_search.max_catalog_items, Some(25));
8227        assert_eq!(
8228            request.runtime.tool_search.provider_variant,
8229            roder_api::inference::ToolSearchProviderVariant::Bm25
8230        );
8231    }
8232
8233    #[tokio::test]
8234    async fn context_entrypoint_hints_use_turn_workspace() {
8235        let process_workspace = runtime_test_workspace("entrypoint-process");
8236        let thread_workspace = runtime_test_workspace("entrypoint-thread");
8237        std::fs::create_dir_all(process_workspace.join("src")).unwrap();
8238        std::fs::create_dir_all(thread_workspace.join("src")).unwrap();
8239        std::fs::write(
8240            process_workspace.join("src/sidebar-thread-groups.ts"),
8241            "export const desktopLeak = true;\n",
8242        )
8243        .unwrap();
8244        std::fs::write(
8245            thread_workspace.join("src/voice-plan-feedback.ts"),
8246            "export const voicePlanFeedback = true;\n",
8247        )
8248        .unwrap();
8249
8250        let captured = Arc::new(StdMutex::new(None));
8251        let mut builder = ExtensionRegistryBuilder::new();
8252        builder.inference_engine(Arc::new(CapturingEngine {
8253            request: captured.clone(),
8254        }));
8255        builder.context_planner(Arc::new(roder_context::EntrypointContextPlanner::new(
8256            process_workspace.clone(),
8257        )));
8258        let runtime = Arc::new(
8259            Runtime::new(
8260                builder.build().unwrap(),
8261                RuntimeConfig {
8262                    workspace: Some(process_workspace.display().to_string()),
8263                    ..RuntimeConfig::default()
8264                },
8265            )
8266            .unwrap(),
8267        );
8268        let mut rx = runtime.subscribe_events();
8269
8270        let turn_id = runtime
8271            .start_turn(StartTurnRequest {
8272                thread_id: "thread-workspace-entrypoint".to_string(),
8273                message: "investigate voice plan feedback".to_string(),
8274                images: Vec::new(),
8275                provider_override: None,
8276                model_override: None,
8277                reasoning_override: None,
8278                workspace: thread_workspace.display().to_string(),
8279                instructions: crate::instructions::default_instructions(),
8280                developer_context: None,
8281                task_ledger_required: false,
8282                service_tier_override: None,
8283            })
8284            .await
8285            .unwrap();
8286
8287        tokio::time::timeout(std::time::Duration::from_secs(5), async {
8288            loop {
8289                let envelope = rx.recv().await.unwrap();
8290                if envelope.turn_id.as_deref() != Some(&turn_id) {
8291                    continue;
8292                }
8293                match envelope.event {
8294                    RoderEvent::TurnCompleted(_) => break,
8295                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8296                    _ => {}
8297                }
8298            }
8299        })
8300        .await
8301        .unwrap();
8302
8303        let request = captured.lock().unwrap().clone().expect("captured request");
8304        let transcript_text = request
8305            .transcript
8306            .iter()
8307            .map(|item| match item {
8308                TranscriptItem::UserMessage(message) => message.text.as_str(),
8309                _ => "",
8310            })
8311            .collect::<Vec<_>>()
8312            .join("\n");
8313        assert!(transcript_text.contains("src/voice-plan-feedback.ts"));
8314        assert!(!transcript_text.contains("src/sidebar-thread-groups.ts"));
8315
8316        let _ = std::fs::remove_dir_all(process_workspace);
8317        let _ = std::fs::remove_dir_all(thread_workspace);
8318    }
8319
8320    fn runtime_test_workspace(name: &str) -> std::path::PathBuf {
8321        let path =
8322            std::env::temp_dir().join(format!("roder-runtime-{name}-{}", uuid::Uuid::new_v4()));
8323        let _ = std::fs::remove_dir_all(&path);
8324        std::fs::create_dir_all(&path).unwrap();
8325        path
8326    }
8327
8328    #[tokio::test]
8329    async fn model_profile_user_model_knobs_override_profile_defaults() {
8330        let request = captured_profile_request(RuntimeConfig {
8331            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8332            default_model: "gpt-5.5".to_string(),
8333            reasoning: Some(REASONING_HIGH.to_string()),
8334            auto_compact_token_limit: Some(999),
8335            model_edit_tools: std::collections::HashMap::from([(
8336                "gpt-5.5".to_string(),
8337                EDIT_TOOL_PATCH.to_string(),
8338            )]),
8339            model_parallel_tool_calls: std::collections::HashMap::from([(
8340                "gpt-5.5".to_string(),
8341                true,
8342            )]),
8343            model_profiles: std::collections::HashMap::from([(
8344                "gpt-5.5".to_string(),
8345                test_model_profile("gpt-5.5"),
8346            )]),
8347            ..RuntimeConfig::default()
8348        })
8349        .await;
8350
8351        let tool_names = request
8352            .tools
8353            .iter()
8354            .map(|tool| tool.name.as_str())
8355            .collect::<Vec<_>>();
8356        assert!(tool_names.contains(&"apply_patch"));
8357        assert!(!tool_names.contains(&"edit"));
8358        assert_eq!(request.reasoning.level.as_deref(), Some(REASONING_HIGH));
8359        assert_eq!(request.runtime.parallel_tool_calls, Some(true));
8360        assert_eq!(request.runtime.auto_compact_token_limit, Some(999));
8361    }
8362
8363    #[tokio::test]
8364    async fn runtime_tool_allowlist_filters_advertised_tools() {
8365        let request = captured_profile_request(RuntimeConfig {
8366            default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8367            default_model: "gpt-5.5".to_string(),
8368            tool_allowlist: vec!["edit".to_string()],
8369            model_profiles: std::collections::HashMap::from([(
8370                "gpt-5.5".to_string(),
8371                test_model_profile("gpt-5.5"),
8372            )]),
8373            ..RuntimeConfig::default()
8374        })
8375        .await;
8376
8377        let tool_names = request
8378            .tools
8379            .iter()
8380            .map(|tool| tool.name.as_str())
8381            .collect::<Vec<_>>();
8382        assert_eq!(tool_names, vec!["edit"]);
8383    }
8384
8385    /// Builds a store-backed runtime turn for a thread created with the given per-thread
8386    /// overrides and returns the captured inference request.
8387    async fn captured_thread_override_request(
8388        runtime: &Arc<Runtime>,
8389        requests: &Arc<StdMutex<Vec<AgentInferenceRequest>>>,
8390        tool_allowlist: Vec<String>,
8391        developer_instructions: Option<String>,
8392        external_tools: Vec<ToolSpec>,
8393    ) -> AgentInferenceRequest {
8394        let thread_id = runtime
8395            .create_thread_with(CreateThreadRequest {
8396                title: Some("Thread overrides".to_string()),
8397                workspace: test_workspace(),
8398                workspace_id: None,
8399                root_id: None,
8400                provider: None,
8401                model: None,
8402                selection_mode: None,
8403                tool_allowlist,
8404                service_tier: None,
8405                developer_instructions,
8406                external_tools,
8407                runner: None,
8408            })
8409            .await
8410            .unwrap()
8411            .thread_id;
8412        let mut rx = runtime.subscribe_events();
8413        let turn_id = runtime
8414            .start_turn(StartTurnRequest {
8415                thread_id,
8416                message: "hello".to_string(),
8417                images: Vec::new(),
8418                provider_override: None,
8419                model_override: None,
8420                reasoning_override: None,
8421                workspace: test_workspace(),
8422                instructions: crate::default_instructions(),
8423                developer_context: None,
8424                task_ledger_required: false,
8425                service_tier_override: None,
8426            })
8427            .await
8428            .unwrap();
8429        tokio::time::timeout(std::time::Duration::from_secs(5), async {
8430            loop {
8431                let envelope = rx.recv().await.unwrap();
8432                if envelope.turn_id.as_deref() != Some(&turn_id) {
8433                    continue;
8434                }
8435                match envelope.event {
8436                    RoderEvent::TurnCompleted(_) => break,
8437                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
8438                    _ => {}
8439                }
8440            }
8441        })
8442        .await
8443        .unwrap();
8444        requests.lock().unwrap().pop().expect("captured request")
8445    }
8446
8447    #[tokio::test]
8448    async fn thread_tool_allowlist_filters_only_that_thread() {
8449        let requests = Arc::new(StdMutex::new(Vec::new()));
8450        let thread_root =
8451            std::env::temp_dir().join(format!("roder-thread-allowlist-{}", uuid::Uuid::new_v4()));
8452        let mut builder = ExtensionRegistryBuilder::new();
8453        builder.inference_engine(Arc::new(SwitchCaptureEngine {
8454            requests: requests.clone(),
8455        }));
8456        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8457            base_path: thread_root.clone(),
8458        }));
8459        builder.tool_contributor(Arc::new(ProfileToolContributor));
8460        let runtime = Arc::new(
8461            Runtime::new(
8462                builder.build().unwrap(),
8463                RuntimeConfig {
8464                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8465                    default_model: "gpt-5.5".to_string(),
8466                    model_profiles: std::collections::HashMap::from([(
8467                        "gpt-5.5".to_string(),
8468                        test_model_profile("gpt-5.5"),
8469                    )]),
8470                    ..RuntimeConfig::default()
8471                },
8472            )
8473            .unwrap(),
8474        );
8475
8476        let allowlisted = captured_thread_override_request(
8477            &runtime,
8478            &requests,
8479            vec!["edit".to_string()],
8480            None,
8481            Vec::new(),
8482        )
8483        .await;
8484        let unrestricted =
8485            captured_thread_override_request(&runtime, &requests, Vec::new(), None, Vec::new())
8486                .await;
8487
8488        let allowlisted_names = allowlisted
8489            .tools
8490            .iter()
8491            .map(|tool| tool.name.as_str())
8492            .collect::<Vec<_>>();
8493        assert_eq!(allowlisted_names, vec!["edit"]);
8494        let unrestricted_names = unrestricted
8495            .tools
8496            .iter()
8497            .map(|tool| tool.name.as_str())
8498            .collect::<Vec<_>>();
8499        assert!(unrestricted_names.contains(&"edit"));
8500        assert!(unrestricted_names.len() > 1);
8501
8502        let _ = std::fs::remove_dir_all(thread_root);
8503    }
8504
8505    /// Builds a store-backed runtime whose `RuntimeConfig.tool_allowlist` is `["edit"]`.
8506    fn runtime_with_edit_allowlist(
8507        requests: &Arc<StdMutex<Vec<AgentInferenceRequest>>>,
8508        thread_root: &std::path::Path,
8509    ) -> Arc<Runtime> {
8510        let mut builder = ExtensionRegistryBuilder::new();
8511        builder.inference_engine(Arc::new(SwitchCaptureEngine {
8512            requests: requests.clone(),
8513        }));
8514        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8515            base_path: thread_root.to_path_buf(),
8516        }));
8517        builder.tool_contributor(Arc::new(ProfileToolContributor));
8518        Arc::new(
8519            Runtime::new(
8520                builder.build().unwrap(),
8521                RuntimeConfig {
8522                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
8523                    default_model: "gpt-5.5".to_string(),
8524                    tool_allowlist: vec!["edit".to_string()],
8525                    model_profiles: std::collections::HashMap::from([(
8526                        "gpt-5.5".to_string(),
8527                        test_model_profile("gpt-5.5"),
8528                    )]),
8529                    ..RuntimeConfig::default()
8530                },
8531            )
8532            .unwrap(),
8533        )
8534    }
8535
8536    #[tokio::test]
8537    async fn runtime_and_thread_allowlists_intersect() {
8538        let requests = Arc::new(StdMutex::new(Vec::new()));
8539        let thread_root = std::env::temp_dir().join(format!(
8540            "roder-allowlist-intersect-{}",
8541            uuid::Uuid::new_v4()
8542        ));
8543        let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8544
8545        let request = captured_thread_override_request(
8546            &runtime,
8547            &requests,
8548            vec!["edit".to_string(), "write_file".to_string()],
8549            None,
8550            Vec::new(),
8551        )
8552        .await;
8553
8554        // The thread allowlist must not re-enable tools the runtime allowlist bans.
8555        let names = request
8556            .tools
8557            .iter()
8558            .map(|tool| tool.name.as_str())
8559            .collect::<Vec<_>>();
8560        assert_eq!(names, vec!["edit"]);
8561
8562        let _ = std::fs::remove_dir_all(thread_root);
8563    }
8564
8565    #[tokio::test]
8566    async fn route_tool_call_denies_tools_outside_allowlists() {
8567        let requests = Arc::new(StdMutex::new(Vec::new()));
8568        let thread_root =
8569            std::env::temp_dir().join(format!("roder-allowlist-dispatch-{}", uuid::Uuid::new_v4()));
8570        let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8571        let thread_id = runtime
8572            .create_thread_with(CreateThreadRequest {
8573                title: Some("Dispatch allowlist".to_string()),
8574                workspace: test_workspace(),
8575                workspace_id: None,
8576                root_id: None,
8577                provider: None,
8578                model: None,
8579                selection_mode: None,
8580                tool_allowlist: vec!["edit".to_string(), "write_file".to_string()],
8581                service_tier: None,
8582                developer_instructions: None,
8583                external_tools: Vec::new(),
8584                runner: None,
8585            })
8586            .await
8587            .unwrap()
8588            .thread_id;
8589
8590        // write_file is registered and on the thread allowlist but banned by the runtime allowlist.
8591        let result = runtime
8592            .route_tool_call(
8593                &thread_id,
8594                &"turn-allowlist-dispatch".to_string(),
8595                roder_api::inference::ToolCallCompleted {
8596                    id: "call-1".to_string(),
8597                    name: "write_file".to_string(),
8598                    arguments: r#"{"path":"a.txt","content":"hi"}"#.to_string(),
8599                },
8600                None,
8601                None,
8602                None,
8603            )
8604            .await
8605            .unwrap();
8606
8607        assert!(result.is_error);
8608        assert!(
8609            result
8610                .result
8611                .contains("not permitted by the tool allowlist"),
8612            "unexpected result: {}",
8613            result.result
8614        );
8615
8616        let _ = std::fs::remove_dir_all(thread_root);
8617    }
8618
8619    #[tokio::test]
8620    async fn route_tool_calls_denies_agent_swarm_mixed_with_other_tools() {
8621        let requests = Arc::new(StdMutex::new(Vec::new()));
8622        let thread_root =
8623            std::env::temp_dir().join(format!("roder-swarm-exclusive-{}", uuid::Uuid::new_v4()));
8624        let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8625
8626        // A response that mixes agent_swarm with another tool is denied wholesale:
8627        // both calls get an error result with retry guidance, and no tool runs.
8628        let results = runtime
8629            .route_tool_calls(
8630                &"thread-swarm".to_string(),
8631                &"turn-swarm".to_string(),
8632                vec![
8633                    roder_api::inference::ToolCallCompleted {
8634                        id: "swarm-1".to_string(),
8635                        name: "agent_swarm".to_string(),
8636                        arguments: "{}".to_string(),
8637                    },
8638                    roder_api::inference::ToolCallCompleted {
8639                        id: "read-1".to_string(),
8640                        name: "read_file".to_string(),
8641                        arguments: "{}".to_string(),
8642                    },
8643                ],
8644                true,
8645                None,
8646                None,
8647            )
8648            .await
8649            .unwrap();
8650
8651        assert_eq!(results.len(), 2, "every tool_call_id must get a response");
8652        assert_eq!(results[0].id, "swarm-1");
8653        assert_eq!(results[1].id, "read-1");
8654        for result in &results {
8655            assert!(result.is_error);
8656            assert!(
8657                result.result.contains("only tool call"),
8658                "unexpected result: {}",
8659                result.result
8660            );
8661        }
8662
8663        let _ = std::fs::remove_dir_all(thread_root);
8664    }
8665
8666    #[tokio::test]
8667    async fn route_tool_calls_emits_agent_swarm_started_event() {
8668        let requests = Arc::new(StdMutex::new(Vec::new()));
8669        let thread_root =
8670            std::env::temp_dir().join(format!("roder-swarm-started-{}", uuid::Uuid::new_v4()));
8671        let runtime = runtime_with_edit_allowlist(&requests, &thread_root);
8672        let mut events = runtime.subscribe_events();
8673
8674        // A single agent_swarm call (the valid shape) emits AgentSwarmStarted on
8675        // the event bus before dispatch, with the child count parsed from args.
8676        let _ = runtime
8677            .route_tool_calls(
8678                &"thread-swarm".to_string(),
8679                &"turn-swarm".to_string(),
8680                vec![roder_api::inference::ToolCallCompleted {
8681                    id: "swarm-1".to_string(),
8682                    name: "agent_swarm".to_string(),
8683                    arguments: r#"{"description":"x","prompt_template":"Read {{item}}","items":["a.rs","b.rs"]}"#
8684                        .to_string(),
8685                }],
8686                true,
8687                None,
8688                None,
8689            )
8690            .await
8691            .unwrap();
8692
8693        let mut started_child_count = None;
8694        for _ in 0..16 {
8695            let envelope = tokio::time::timeout(std::time::Duration::from_secs(2), events.recv())
8696                .await
8697                .unwrap()
8698                .unwrap();
8699            if let RoderEvent::AgentSwarmStarted(event) = envelope.event {
8700                started_child_count = Some(event.child_count);
8701                assert_eq!(event.tool_id, "swarm-1");
8702                break;
8703            }
8704        }
8705        assert_eq!(started_child_count, Some(2));
8706
8707        let _ = std::fs::remove_dir_all(thread_root);
8708    }
8709
8710    #[test]
8711    fn agent_swarm_child_count_sums_items_and_resumes() {
8712        assert_eq!(
8713            agent_swarm_child_count(r#"{"description":"x","items":["a","b","c"]}"#),
8714            3
8715        );
8716        assert_eq!(
8717            agent_swarm_child_count(
8718                r#"{"description":"x","items":["a"],"resume_agent_ids":{"id1":"continue"}}"#
8719            ),
8720            2
8721        );
8722        // Malformed input is lenient.
8723        assert_eq!(agent_swarm_child_count("not json"), 0);
8724    }
8725
8726    #[test]
8727    fn parse_swarm_counts_reads_summary_with_omitted_buckets() {
8728        let text = "<agent_swarm_result>\n<summary>completed: 2, failed: 1</summary>\n</agent_swarm_result>";
8729        assert_eq!(parse_swarm_counts(text), Some((2, 1, 0)));
8730        let text = "<agent_swarm_result>\n<summary>completed: 0</summary>\n</agent_swarm_result>";
8731        assert_eq!(parse_swarm_counts(text), Some((0, 0, 0)));
8732        // Not a swarm result.
8733        assert_eq!(parse_swarm_counts("just text"), None);
8734    }
8735
8736    /// Signals when inference starts, then waits for `proceed` and fails the stream.
8737    struct SignalledFailureEngine {
8738        started: tokio::sync::mpsc::UnboundedSender<()>,
8739        proceed: Arc<tokio::sync::Notify>,
8740    }
8741
8742    #[async_trait::async_trait]
8743    impl InferenceEngine for SignalledFailureEngine {
8744        fn id(&self) -> String {
8745            roder_api::catalog::PROVIDER_MOCK.to_string()
8746        }
8747
8748        fn capabilities(&self) -> InferenceCapabilities {
8749            InferenceCapabilities::coding_agent_default()
8750        }
8751
8752        async fn list_models(
8753            &self,
8754            _ctx: InferenceProviderContext<'_>,
8755        ) -> anyhow::Result<Vec<roder_api::inference::ModelDescriptor>> {
8756            Ok(roder_api::catalog::models_for_provider(
8757                roder_api::catalog::PROVIDER_MOCK,
8758                true,
8759            ))
8760        }
8761
8762        async fn stream_turn(
8763            &self,
8764            _ctx: InferenceTurnContext<'_>,
8765            _request: AgentInferenceRequest,
8766        ) -> anyhow::Result<InferenceEventStream> {
8767            let _ = self.started.send(());
8768            self.proceed.notified().await;
8769            anyhow::bail!("engine failed mid-turn")
8770        }
8771    }
8772
8773    #[tokio::test]
8774    async fn failed_turn_sweeps_pending_external_tool_calls() {
8775        let (started_tx, mut started_rx) = tokio::sync::mpsc::unbounded_channel();
8776        let proceed = Arc::new(tokio::sync::Notify::new());
8777        let mut builder = ExtensionRegistryBuilder::new();
8778        builder.inference_engine(Arc::new(SignalledFailureEngine {
8779            started: started_tx,
8780            proceed: proceed.clone(),
8781        }));
8782        let runtime =
8783            Arc::new(Runtime::new(builder.build().unwrap(), RuntimeConfig::default()).unwrap());
8784        let mut rx = runtime.subscribe_events();
8785        let turn_id = runtime
8786            .start_turn(StartTurnRequest {
8787                thread_id: "thread-sweep".to_string(),
8788                message: "go".to_string(),
8789                images: Vec::new(),
8790                provider_override: None,
8791                model_override: None,
8792                reasoning_override: None,
8793                workspace: test_workspace(),
8794                instructions: InstructionBundle {
8795                    system: None,
8796                    developer: None,
8797                    developer_context: None,
8798                },
8799                developer_context: None,
8800                task_ledger_required: false,
8801                service_tier_override: None,
8802            })
8803            .await
8804            .unwrap();
8805        tokio::time::timeout(std::time::Duration::from_secs(5), started_rx.recv())
8806            .await
8807            .unwrap()
8808            .unwrap();
8809
8810        let (tx, _pending_rx) = oneshot::channel();
8811        runtime.pending_external_tool_calls.lock().await.insert(
8812            "exttool-sweep-test".to_string(),
8813            PendingExternalToolCall {
8814                thread_id: "thread-sweep".to_string(),
8815                turn_id: turn_id.clone(),
8816                tool_id: "call-1".to_string(),
8817                tool_name: "acme_lookup".to_string(),
8818                tx,
8819            },
8820        );
8821        proceed.notify_one();
8822
8823        let outcome = tokio::time::timeout(std::time::Duration::from_secs(5), async {
8824            loop {
8825                let envelope = rx.recv().await.unwrap();
8826                if let RoderEvent::ExternalToolCallResolved(event) = envelope.event
8827                    && event.request_id == "exttool-sweep-test"
8828                {
8829                    break event.outcome;
8830                }
8831            }
8832        })
8833        .await
8834        .expect("turn failure must resolve pending external tool calls");
8835        assert_eq!(outcome, ExternalToolCallOutcome::Cancelled);
8836        assert!(runtime.pending_external_tool_calls.lock().await.is_empty());
8837    }
8838
8839    #[tokio::test]
8840    async fn thread_developer_instructions_layer_under_harness_prompt() {
8841        let requests = Arc::new(StdMutex::new(Vec::new()));
8842        let thread_root = std::env::temp_dir().join(format!(
8843            "roder-thread-instructions-{}",
8844            uuid::Uuid::new_v4()
8845        ));
8846        let mut builder = ExtensionRegistryBuilder::new();
8847        builder.inference_engine(Arc::new(SwitchCaptureEngine {
8848            requests: requests.clone(),
8849        }));
8850        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8851            base_path: thread_root.clone(),
8852        }));
8853        builder.tool_contributor(Arc::new(ProfileToolContributor));
8854        let runtime =
8855            Arc::new(Runtime::new(builder.build().unwrap(), RuntimeConfig::default()).unwrap());
8856
8857        let request = captured_thread_override_request(
8858            &runtime,
8859            &requests,
8860            Vec::new(),
8861            Some("You are embedded in a host app.".to_string()),
8862            Vec::new(),
8863        )
8864        .await;
8865
8866        let system = request.instructions.system.expect("system instructions");
8867        assert!(system.starts_with("You are Roder"));
8868        let developer = request
8869            .instructions
8870            .developer
8871            .expect("developer instructions");
8872        assert!(developer.starts_with("You are embedded in a host app."));
8873
8874        let plain =
8875            captured_thread_override_request(&runtime, &requests, Vec::new(), None, Vec::new())
8876                .await;
8877        assert_eq!(plain.instructions.developer, None);
8878
8879        let _ = std::fs::remove_dir_all(thread_root);
8880    }
8881
8882    #[tokio::test]
8883    async fn thread_external_tools_are_advertised_and_shadow_builtins() {
8884        let requests = Arc::new(StdMutex::new(Vec::new()));
8885        let thread_root = std::env::temp_dir().join(format!(
8886            "roder-thread-external-tools-{}",
8887            uuid::Uuid::new_v4()
8888        ));
8889        let mut builder = ExtensionRegistryBuilder::new();
8890        builder.inference_engine(Arc::new(SwitchCaptureEngine {
8891            requests: requests.clone(),
8892        }));
8893        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8894            base_path: thread_root.clone(),
8895        }));
8896        builder.tool_contributor(Arc::new(ProfileToolContributor));
8897        let runtime =
8898            Arc::new(Runtime::new(builder.build().unwrap(), RuntimeConfig::default()).unwrap());
8899
8900        let external_tools = vec![
8901            ToolSpec {
8902                name: "acme_lookup".to_string(),
8903                description: "Look up Acme workspace state.".to_string(),
8904                parameters: serde_json::json!({
8905                    "type": "object",
8906                    "properties": { "query": { "type": "string" } },
8907                    "required": ["query"]
8908                }),
8909            },
8910            ToolSpec {
8911                name: "edit".to_string(),
8912                description: "Host-managed edit.".to_string(),
8913                parameters: serde_json::json!({ "type": "object" }),
8914            },
8915        ];
8916        let request =
8917            captured_thread_override_request(&runtime, &requests, Vec::new(), None, external_tools)
8918                .await;
8919
8920        let acme = request
8921            .tools
8922            .iter()
8923            .find(|tool| tool.name == "acme_lookup")
8924            .expect("external tool advertised");
8925        assert_eq!(acme.description, "Look up Acme workspace state.");
8926        assert_eq!(acme.parameters["required"][0], "query");
8927        let edits = request
8928            .tools
8929            .iter()
8930            .filter(|tool| tool.name == "edit")
8931            .collect::<Vec<_>>();
8932        assert_eq!(edits.len(), 1, "external edit shadows the builtin");
8933        assert_eq!(edits[0].description, "Host-managed edit.");
8934
8935        let plain =
8936            captured_thread_override_request(&runtime, &requests, Vec::new(), None, Vec::new())
8937                .await;
8938        assert!(plain.tools.iter().all(|tool| tool.name != "acme_lookup"));
8939        let plain_edit = plain
8940            .tools
8941            .iter()
8942            .find(|tool| tool.name == "edit")
8943            .expect("builtin edit advertised on plain thread");
8944        assert_eq!(plain_edit.description, "edit test tool");
8945
8946        let _ = std::fs::remove_dir_all(thread_root);
8947    }
8948
8949    #[tokio::test]
8950    async fn model_selection_reads_and_updates_metadata_without_loading_history() {
8951        let thread_root = tempfile::tempdir().unwrap();
8952        let mut builder = ExtensionRegistryBuilder::new();
8953        builder.inference_engine(Arc::new(SwitchCaptureEngine {
8954            requests: Arc::new(StdMutex::new(Vec::new())),
8955        }));
8956        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
8957            base_path: thread_root.path().to_path_buf(),
8958        }));
8959        let runtime = Runtime::new(builder.build().unwrap(), RuntimeConfig::default()).unwrap();
8960        let thread = runtime
8961            .create_thread(Some("Metadata-only model switch".to_string()))
8962            .await
8963            .unwrap();
8964        let dir = thread_root.path().join(&thread.thread_id);
8965        // Invalid history makes any accidental full-snapshot read fail.
8966        for file in ["events.jsonl", "item_events.jsonl", "extension_state.jsonl"] {
8967            tokio::fs::write(dir.join(file), b"history must not be read\n")
8968                .await
8969                .unwrap();
8970        }
8971        let selection = ModelSelectionMode::manual(
8972            roder_api::catalog::PROVIDER_MOCK.to_string(),
8973            "gpt-5.5".to_string(),
8974            Some("medium".to_string()),
8975        );
8976
8977        runtime
8978            .set_thread_selection_mode(&thread.thread_id, selection.clone())
8979            .await
8980            .unwrap();
8981        assert_eq!(
8982            runtime
8983                .selection_mode_for_thread(&thread.thread_id)
8984                .await
8985                .unwrap(),
8986            Some(selection.clone())
8987        );
8988        let metadata = runtime
8989            .load_thread_metadata(&thread.thread_id)
8990            .await
8991            .unwrap()
8992            .unwrap();
8993        assert_eq!(metadata.selection_mode, Some(selection));
8994        assert_eq!(metadata.provider, thread.provider);
8995        assert_eq!(metadata.model.as_deref(), Some("gpt-5.5"));
8996        assert_eq!(metadata.title, thread.title);
8997        assert_eq!(metadata.workspace, thread.workspace);
8998        for file in ["events.jsonl", "item_events.jsonl", "extension_state.jsonl"] {
8999            assert_eq!(
9000                tokio::fs::read(dir.join(file)).await.unwrap(),
9001                b"history must not be read\n"
9002            );
9003        }
9004        assert!(
9005            runtime
9006                .set_thread_selection_mode(
9007                    &"missing-thread".to_string(),
9008                    ModelSelectionMode::manual("mock", "mock", None),
9009                )
9010                .await
9011                .unwrap_err()
9012                .to_string()
9013                .contains("thread not found")
9014        );
9015    }
9016
9017    #[tokio::test]
9018    async fn model_switch_injects_summary_and_records_profile_segments() {
9019        let requests = Arc::new(StdMutex::new(Vec::new()));
9020        let thread_root = std::env::temp_dir().join(format!(
9021            "roder-model-switch-thread-{}",
9022            uuid::Uuid::new_v4()
9023        ));
9024        let mut builder = ExtensionRegistryBuilder::new();
9025        builder.inference_engine(Arc::new(SwitchCaptureEngine {
9026            requests: requests.clone(),
9027        }));
9028        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
9029            base_path: thread_root.clone(),
9030        }));
9031        builder.tool_contributor(Arc::new(ProfileToolContributor));
9032        let mut claude_profile = test_model_profile("claude-haiku-4-5-20251001");
9033        claude_profile.provider_family = ProviderFamily::Anthropic;
9034        claude_profile.edit_tool = Some(EDIT_TOOL_EDIT.to_string());
9035        let runtime = Arc::new(
9036            Runtime::new(
9037                builder.build().unwrap(),
9038                RuntimeConfig {
9039                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9040                    default_model: "gpt-5.5".to_string(),
9041                    model_profiles: std::collections::HashMap::from([
9042                        ("gpt-5.5".to_string(), test_model_profile("gpt-5.5")),
9043                        ("claude-haiku-4-5-20251001".to_string(), claude_profile),
9044                    ]),
9045                    ..RuntimeConfig::default()
9046                },
9047            )
9048            .unwrap(),
9049        );
9050        let thread_id = runtime
9051            .create_thread_with(CreateThreadRequest {
9052                title: Some("Model switch".to_string()),
9053                workspace: test_workspace(),
9054                workspace_id: None,
9055                root_id: None,
9056                provider: None,
9057                model: None,
9058                selection_mode: None,
9059                tool_allowlist: Vec::new(),
9060                service_tier: None,
9061                developer_instructions: None,
9062                external_tools: Vec::new(),
9063                runner: None,
9064            })
9065            .await
9066            .unwrap()
9067            .thread_id;
9068        let mut rx = runtime.subscribe_events();
9069        for (message, model_override) in [
9070            ("first turn", None),
9071            ("second turn", Some("claude-haiku-4-5-20251001".to_string())),
9072        ] {
9073            let turn_id = runtime
9074                .start_turn(StartTurnRequest {
9075                    thread_id: thread_id.clone(),
9076                    message: message.to_string(),
9077                    images: Vec::new(),
9078                    provider_override: None,
9079                    model_override,
9080                    reasoning_override: None,
9081                    workspace: test_workspace(),
9082                    instructions: InstructionBundle::default(),
9083                    developer_context: None,
9084                    task_ledger_required: false,
9085                    service_tier_override: None,
9086                })
9087                .await
9088                .unwrap();
9089            tokio::time::timeout(std::time::Duration::from_secs(5), async {
9090                loop {
9091                    let envelope = rx.recv().await.unwrap();
9092                    if envelope.turn_id.as_deref() != Some(&turn_id) {
9093                        continue;
9094                    }
9095                    match envelope.event {
9096                        RoderEvent::TurnCompleted(_) => break,
9097                        RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9098                        _ => {}
9099                    }
9100                }
9101            })
9102            .await
9103            .unwrap();
9104        }
9105
9106        let captured = requests.lock().unwrap().clone();
9107        assert_eq!(captured.len(), 2);
9108        assert!(captured[1].transcript.iter().any(|item| {
9109            matches!(
9110                item,
9111                TranscriptItem::UserMessage(message)
9112                    if message.text.starts_with(MODEL_SWITCH_SUMMARY_PREFIX)
9113                        && message.text.contains("previous profile mock/gpt-5.5")
9114                        && message.text.contains("Current profile mock/claude-haiku-4-5-20251001")
9115                        && message.text.contains("Available tools now:")
9116            )
9117        }));
9118
9119        let snapshot = runtime
9120            .thread_store
9121            .as_ref()
9122            .unwrap()
9123            .load_thread(&thread_id)
9124            .await
9125            .unwrap()
9126            .unwrap();
9127        let trace_segments = snapshot
9128            .turns
9129            .iter()
9130            .flat_map(|turn| &turn.items)
9131            .filter(|item| {
9132                matches!(
9133                    item,
9134                    TranscriptItem::ProviderMetadata(value)
9135                        if value.get("kind").and_then(serde_json::Value::as_str)
9136                            == Some(MODEL_PROFILE_TRACE_KIND)
9137                            && value.get("segment").and_then(serde_json::Value::as_str)
9138                                == Some("assistant")
9139                )
9140            })
9141            .count();
9142        assert!(trace_segments >= 2);
9143        let _ = std::fs::remove_dir_all(thread_root);
9144    }
9145
9146    struct CountingTaskTool {
9147        calls: Arc<StdMutex<u32>>,
9148    }
9149
9150    #[async_trait::async_trait]
9151    impl ToolExecutor for CountingTaskTool {
9152        fn spec(&self) -> ToolSpec {
9153            ToolSpec {
9154                name: "task".to_string(),
9155                description: "Dispatch a test subagent.".to_string(),
9156                parameters: serde_json::json!({
9157                    "type": "object",
9158                    "properties": {
9159                        "description": { "type": "string" },
9160                        "prompt": { "type": "string" },
9161                        "parent_deadline_seconds": { "type": "integer" }
9162                    },
9163                    "required": ["description", "prompt"],
9164                    "additionalProperties": false
9165                }),
9166            }
9167        }
9168
9169        async fn execute(
9170            &self,
9171            _ctx: ToolExecutionContext,
9172            call: ToolCall,
9173        ) -> anyhow::Result<ToolResult> {
9174            *self.calls.lock().unwrap() += 1;
9175            Ok(ToolResult {
9176                id: call.id,
9177                name: call.name,
9178                text: "started child".to_string(),
9179                data: serde_json::json!({}),
9180                is_error: false,
9181            })
9182        }
9183    }
9184
9185    #[tokio::test]
9186    async fn runtime_profile_reaches_inference_request_and_turn_metadata() {
9187        let captured = Arc::new(StdMutex::new(None));
9188        let mut builder = ExtensionRegistryBuilder::new();
9189        builder.inference_engine(Arc::new(CapturingEngine {
9190            request: captured.clone(),
9191        }));
9192        let runtime = Arc::new(
9193            Runtime::new(
9194                builder.build().unwrap(),
9195                RuntimeConfig {
9196                    runtime_profile: RuntimeProfile::NonInteractive,
9197                    ..RuntimeConfig::default()
9198                },
9199            )
9200            .unwrap(),
9201        );
9202        let mut rx = runtime.subscribe_events();
9203        let turn_id = runtime
9204            .start_turn(StartTurnRequest {
9205                thread_id: "thread-profile".to_string(),
9206                message: "work unattended".to_string(),
9207                images: Vec::new(),
9208                provider_override: None,
9209                model_override: None,
9210                reasoning_override: None,
9211                workspace: test_workspace(),
9212                instructions: InstructionBundle {
9213                    system: None,
9214                    developer: Some("base developer".to_string()),
9215                    developer_context: None,
9216                },
9217                developer_context: None,
9218                task_ledger_required: false,
9219                service_tier_override: None,
9220            })
9221            .await
9222            .unwrap();
9223
9224        let mut observed_profile = None;
9225        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9226            loop {
9227                let envelope = rx.recv().await.unwrap();
9228                if envelope.turn_id.as_deref() != Some(&turn_id) {
9229                    continue;
9230                }
9231                match envelope.event {
9232                    RoderEvent::TurnStarted(event) => {
9233                        observed_profile = Some(event.runtime_profile);
9234                    }
9235                    RoderEvent::TurnCompleted(_) => break,
9236                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9237                    _ => {}
9238                }
9239            }
9240        })
9241        .await
9242        .unwrap();
9243
9244        assert_eq!(observed_profile, Some(RuntimeProfile::NonInteractive));
9245        let request = captured.lock().unwrap().clone().unwrap();
9246        assert_eq!(request.runtime.profile, RuntimeProfile::NonInteractive);
9247        let developer = request.instructions.developer.unwrap();
9248        assert!(developer.contains("base developer"));
9249        assert!(developer.contains("non-interactive profile"));
9250    }
9251
9252    #[tokio::test]
9253    async fn global_policy_mode_changes_do_not_create_runtime_thread_directory() {
9254        let workspace = runtime_test_workspace("global-policy-mode");
9255        let thread_root = workspace.join("threads");
9256        let mut builder = ExtensionRegistryBuilder::new();
9257        builder.inference_engine(Arc::new(FakeInferenceEngine));
9258        builder.thread_store_factory(Arc::new(JsonlThreadStoreFactory {
9259            base_path: thread_root.clone(),
9260        }));
9261        let runtime = Runtime::new(
9262            builder.build().unwrap(),
9263            RuntimeConfig {
9264                workspace: Some(workspace.display().to_string()),
9265                ..Default::default()
9266            },
9267        )
9268        .unwrap();
9269
9270        runtime
9271            .set_policy_mode(PolicyMode::AcceptAll, Some("test".to_string()))
9272            .await
9273            .unwrap();
9274
9275        assert!(!thread_root.join("runtime").exists());
9276        let _ = std::fs::remove_dir_all(workspace);
9277    }
9278
9279    #[tokio::test]
9280    async fn task_ledger_enforcement_injects_eval_reminder_before_work() {
9281        let captured = Arc::new(StdMutex::new(None));
9282        let mut builder = ExtensionRegistryBuilder::new();
9283        builder.inference_engine(Arc::new(CapturingEngine {
9284            request: captured.clone(),
9285        }));
9286        builder.tool_contributor(Arc::new(
9287            roder_ext_task_ledger::TaskLedgerToolContributor::default(),
9288        ));
9289        let runtime = Arc::new(
9290            Runtime::new(
9291                builder.build().unwrap(),
9292                RuntimeConfig {
9293                    runtime_profile: RuntimeProfile::Eval,
9294                    policy_mode: PolicyMode::Bypass,
9295                    agent_swarm_mode: false,
9296                    ultra_mode: false,
9297                    ..RuntimeConfig::default()
9298                },
9299            )
9300            .unwrap(),
9301        );
9302        let mut rx = runtime.subscribe_events();
9303        let turn_id = runtime
9304            .start_turn(StartTurnRequest {
9305                thread_id: "thread-ledger".to_string(),
9306                message: "decomposed work".to_string(),
9307                images: Vec::new(),
9308                provider_override: None,
9309                model_override: None,
9310                reasoning_override: None,
9311                workspace: test_workspace(),
9312                instructions: InstructionBundle::default(),
9313                developer_context: None,
9314                task_ledger_required: true,
9315                service_tier_override: None,
9316            })
9317            .await
9318            .unwrap();
9319
9320        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9321            loop {
9322                let envelope = rx.recv().await.unwrap();
9323                if envelope.turn_id.as_deref() == Some(&turn_id)
9324                    && matches!(envelope.event, RoderEvent::TurnCompleted(_))
9325                {
9326                    break;
9327                }
9328            }
9329        })
9330        .await
9331        .unwrap();
9332
9333        let request = captured.lock().unwrap().clone().unwrap();
9334        let developer = request.instructions.developer.unwrap();
9335        assert!(developer.contains("Task Ledger Required"));
9336        assert!(developer.contains("task_ledger.update"));
9337        let tool_names: Vec<_> = request
9338            .tools
9339            .iter()
9340            .map(|tool| tool.name.as_str())
9341            .collect();
9342        assert!(
9343            tool_names.contains(&TASK_LEDGER_TOOL_NAME),
9344            "tool names: {tool_names:?}"
9345        );
9346        assert_eq!(
9347            request.tool_choice,
9348            ToolChoice::Specific(TASK_LEDGER_TOOL_NAME.to_string())
9349        );
9350        assert_eq!(request.tools.len(), 1);
9351        assert_eq!(request.tools[0].name, TASK_LEDGER_TOOL_NAME);
9352    }
9353
9354    #[tokio::test]
9355    async fn eval_task_ledger_blocks_final_answer_until_open_items_are_completed() {
9356        let requests = Arc::new(StdMutex::new(Vec::new()));
9357        let mut builder = ExtensionRegistryBuilder::new();
9358        builder.inference_engine(Arc::new(TaskLedgerCompletionGateEngine {
9359            calls: StdMutex::new(0),
9360            requests: requests.clone(),
9361        }));
9362        builder.tool_contributor(Arc::new(
9363            roder_ext_task_ledger::TaskLedgerToolContributor::default(),
9364        ));
9365        let runtime = Arc::new(
9366            Runtime::new(
9367                builder.build().unwrap(),
9368                RuntimeConfig {
9369                    runtime_profile: RuntimeProfile::Eval,
9370                    policy_mode: PolicyMode::Bypass,
9371                    agent_swarm_mode: false,
9372                    ultra_mode: false,
9373                    ..RuntimeConfig::default()
9374                },
9375            )
9376            .unwrap(),
9377        );
9378        let mut rx = runtime.subscribe_events();
9379        let turn_id = runtime
9380            .start_turn(StartTurnRequest {
9381                thread_id: "thread-ledger-completion".to_string(),
9382                message: "write the answer file".to_string(),
9383                images: Vec::new(),
9384                provider_override: None,
9385                model_override: None,
9386                reasoning_override: None,
9387                workspace: test_workspace(),
9388                instructions: InstructionBundle::default(),
9389                developer_context: None,
9390                task_ledger_required: true,
9391                service_tier_override: None,
9392            })
9393            .await
9394            .unwrap();
9395
9396        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9397            loop {
9398                let envelope = rx.recv().await.unwrap();
9399                if envelope.turn_id.as_deref() != Some(&turn_id) {
9400                    continue;
9401                }
9402                match envelope.event {
9403                    RoderEvent::TurnCompleted(_) => break,
9404                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9405                    _ => {}
9406                }
9407            }
9408        })
9409        .await
9410        .unwrap();
9411
9412        let requests = requests.lock().unwrap().clone();
9413        assert_eq!(requests.len(), 4);
9414        assert!(requests[2].transcript.iter().any(|item| {
9415            matches!(
9416                item,
9417                TranscriptItem::UserMessage(message)
9418                    if message.text.contains("Task Ledger Completion Required")
9419                        && message.text.contains("Write /app/result.txt")
9420            )
9421        }));
9422        assert!(requests[3].transcript.iter().any(|item| {
9423            matches!(
9424                item,
9425                TranscriptItem::ToolResult(result)
9426                    if result.name.as_deref() == Some(TASK_LEDGER_TOOL_NAME)
9427                        && result.result.contains("Task ledger: 2/2 completed")
9428            )
9429        }));
9430    }
9431
9432    #[tokio::test]
9433    async fn eval_task_ledger_checkpoint_requests_scoreable_file_before_final_reserve() {
9434        let requests = Arc::new(StdMutex::new(Vec::new()));
9435        let mut builder = ExtensionRegistryBuilder::new();
9436        builder.inference_engine(Arc::new(TaskLedgerCompletionGateEngine {
9437            calls: StdMutex::new(0),
9438            requests: requests.clone(),
9439        }));
9440        builder.tool_contributor(Arc::new(
9441            roder_ext_task_ledger::TaskLedgerToolContributor::default(),
9442        ));
9443        let runtime = Arc::new(
9444            Runtime::new(
9445                builder.build().unwrap(),
9446                RuntimeConfig {
9447                    runtime_profile: RuntimeProfile::Eval,
9448                    policy_mode: PolicyMode::Bypass,
9449                    agent_swarm_mode: false,
9450                    ultra_mode: false,
9451                    turn_deadline_seconds: Some(120),
9452                    ..RuntimeConfig::default()
9453                },
9454            )
9455            .unwrap(),
9456        );
9457        let mut rx = runtime.subscribe_events();
9458        let turn_id = runtime
9459            .start_turn(StartTurnRequest {
9460                thread_id: "thread-ledger-checkpoint".to_string(),
9461                message: "write the answer file".to_string(),
9462                images: Vec::new(),
9463                provider_override: None,
9464                model_override: None,
9465                reasoning_override: None,
9466                workspace: test_workspace(),
9467                instructions: InstructionBundle::default(),
9468                developer_context: None,
9469                task_ledger_required: true,
9470                service_tier_override: None,
9471            })
9472            .await
9473            .unwrap();
9474
9475        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9476            loop {
9477                let envelope = rx.recv().await.unwrap();
9478                if envelope.turn_id.as_deref() != Some(&turn_id) {
9479                    continue;
9480                }
9481                match envelope.event {
9482                    RoderEvent::TurnCompleted(_) => break,
9483                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9484                    _ => {}
9485                }
9486            }
9487        })
9488        .await
9489        .unwrap();
9490
9491        let requests = requests.lock().unwrap().clone();
9492        assert!(requests.len() >= 2);
9493        assert!(requests[1].transcript.iter().any(|item| {
9494            matches!(
9495                item,
9496                TranscriptItem::UserMessage(message)
9497                    if message.text.contains("Scoreable Output Checkpoint")
9498                        && message.text.contains("ensure the required output file(s) exist")
9499                        && message.text.contains("Write /app/result.txt")
9500            )
9501        }));
9502    }
9503
9504    #[test]
9505    fn deadline_task_ledger_prompt_preserves_scoreable_work_instruction() {
9506        let prompt = task_ledger_deadline_completion_prompt(
9507            12,
9508            30,
9509            "Task Ledger Completion Required: write /app/result.txt, then call task_ledger.update",
9510        );
9511
9512        assert!(prompt.contains("12 seconds remain"));
9513        assert!(prompt.contains("create or update the required scoreable output files"));
9514        assert!(prompt.contains("write /app/result.txt"));
9515        assert!(prompt.contains(TASK_LEDGER_TOOL_NAME));
9516    }
9517
9518    #[test]
9519    fn scoreable_checkpoint_prompt_preserves_provisional_file_instruction() {
9520        let prompt = task_ledger_scoreable_checkpoint_prompt(
9521            120,
9522            "Task Ledger Completion Required: write /app/result.txt, then call task_ledger.update",
9523        );
9524
9525        assert!(prompt.contains("120 seconds remain"));
9526        assert!(prompt.contains("best evidence-backed answer"));
9527        assert!(prompt.contains("even if provisional"));
9528        assert!(prompt.contains("preserve that candidate"));
9529        assert!(prompt.contains("partial-coverage"));
9530        assert!(prompt.contains("write /app/result.txt"));
9531        assert!(prompt.contains(TASK_LEDGER_TOOL_NAME));
9532    }
9533
9534    #[test]
9535    fn open_task_ledger_moves_inference_timeout_to_scoreable_checkpoint() {
9536        let deadline = Some(OffsetDateTime::now_utc() + Duration::seconds(870));
9537        let transcript = vec![TranscriptItem::ToolResult(ToolResultRecord {
9538            id: "ledger-open".to_string(),
9539            name: Some(TASK_LEDGER_TOOL_NAME.to_string()),
9540            result: "Task ledger: 0/1 completed\n- pending: Write /app/result.txt [write]"
9541                .to_string(),
9542            display_payload: None,
9543            is_error: false,
9544        })];
9545
9546        let (_, action) = inference_timeout_deadline(
9547            deadline,
9548            RuntimeProfile::Eval,
9549            true,
9550            30,
9551            false,
9552            0,
9553            &transcript,
9554        )
9555        .unwrap();
9556
9557        assert_eq!(action, InferenceTimeoutAction::ScoreableCheckpoint);
9558    }
9559
9560    #[tokio::test]
9561    async fn verification_gate_forces_eval_code_changes_through_review() {
9562        let mut builder = ExtensionRegistryBuilder::new();
9563        builder.inference_engine(Arc::new(VerificationGateEngine {
9564            calls: StdMutex::new(0),
9565        }));
9566        builder.tool_contributor(Arc::new(WriteFileContributor));
9567        builder.tool_contributor(Arc::new(
9568            roder_ext_verification::VerificationToolContributor,
9569        ));
9570        let runtime = Arc::new(
9571            Runtime::new(
9572                builder.build().unwrap(),
9573                RuntimeConfig {
9574                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9575                    default_model: "mock".to_string(),
9576                    runtime_profile: RuntimeProfile::Eval,
9577                    policy_mode: PolicyMode::Bypass,
9578                    agent_swarm_mode: false,
9579                    ultra_mode: false,
9580                    ..RuntimeConfig::default()
9581                },
9582            )
9583            .unwrap(),
9584        );
9585        let mut rx = runtime.subscribe_events();
9586        let turn_id = runtime
9587            .start_turn(StartTurnRequest {
9588                thread_id: "thread-verification".to_string(),
9589                message: "write code".to_string(),
9590                images: Vec::new(),
9591                provider_override: None,
9592                model_override: None,
9593                reasoning_override: None,
9594                workspace: test_workspace(),
9595                instructions: InstructionBundle::default(),
9596                developer_context: None,
9597                task_ledger_required: false,
9598                service_tier_override: None,
9599            })
9600            .await
9601            .unwrap();
9602
9603        let mut saw_required = false;
9604        let mut saw_completed = false;
9605        let mut final_text = String::new();
9606        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9607            loop {
9608                let envelope = rx.recv().await.unwrap();
9609                if envelope.turn_id.as_deref() != Some(&turn_id) {
9610                    continue;
9611                }
9612                match envelope.event {
9613                    RoderEvent::VerificationRequired(event) => {
9614                        saw_required = true;
9615                        assert_eq!(event.changed_files, vec!["src/lib.rs"]);
9616                    }
9617                    RoderEvent::VerificationCompleted(event) => {
9618                        saw_completed = true;
9619                        assert!(event.passed);
9620                    }
9621                    RoderEvent::InferenceEventReceived(event) => {
9622                        if let InferenceEvent::MessageDelta(delta) = event.event {
9623                            final_text.push_str(&delta.text);
9624                        }
9625                    }
9626                    RoderEvent::TurnCompleted(_) => break,
9627                    _ => {}
9628                }
9629            }
9630        })
9631        .await
9632        .unwrap();
9633
9634        assert!(saw_required);
9635        assert!(saw_completed);
9636        assert!(final_text.contains("verified final"));
9637    }
9638
9639    #[tokio::test]
9640    async fn speed_policy_changes_reasoning_across_eval_model_calls_without_model_switch() {
9641        let requests = Arc::new(StdMutex::new(Vec::new()));
9642        let mut builder = ExtensionRegistryBuilder::new();
9643        builder.inference_engine(Arc::new(SpeedPolicyEngine {
9644            calls: StdMutex::new(0),
9645            requests: requests.clone(),
9646        }));
9647        builder.tool_contributor(Arc::new(WriteFileContributor));
9648        builder.tool_contributor(Arc::new(
9649            roder_ext_verification::VerificationToolContributor,
9650        ));
9651        let runtime = Arc::new(
9652            Runtime::new(
9653                builder.build().unwrap(),
9654                RuntimeConfig {
9655                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9656                    default_model: "gpt-5.5".to_string(),
9657                    runtime_profile: RuntimeProfile::Eval,
9658                    policy_mode: PolicyMode::Bypass,
9659                    agent_swarm_mode: false,
9660                    ultra_mode: false,
9661                    ..RuntimeConfig::default()
9662                },
9663            )
9664            .unwrap(),
9665        );
9666        let mut rx = runtime.subscribe_events();
9667        let turn_id = runtime
9668            .start_turn(StartTurnRequest {
9669                thread_id: "thread-speed-policy".to_string(),
9670                message: "write code".to_string(),
9671                images: Vec::new(),
9672                provider_override: None,
9673                model_override: None,
9674                reasoning_override: None,
9675                workspace: test_workspace(),
9676                instructions: InstructionBundle::default(),
9677                developer_context: None,
9678                task_ledger_required: false,
9679                service_tier_override: None,
9680            })
9681            .await
9682            .unwrap();
9683
9684        let mut saw_speed_policy_event = false;
9685        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9686            loop {
9687                let envelope = rx.recv().await.unwrap();
9688                if envelope.turn_id.as_deref() != Some(&turn_id) {
9689                    continue;
9690                }
9691                match envelope.event {
9692                    RoderEvent::InferenceStarted(event) => {
9693                        if event.speed_policy.is_some() {
9694                            saw_speed_policy_event = true;
9695                        }
9696                    }
9697                    RoderEvent::TurnCompleted(_) => break,
9698                    RoderEvent::TurnFailed(event) => panic!("turn failed: {}", event.error),
9699                    _ => {}
9700                }
9701            }
9702        })
9703        .await
9704        .unwrap();
9705
9706        let requests = requests.lock().unwrap().clone();
9707        assert!(saw_speed_policy_event);
9708        assert!(requests.len() >= 4);
9709        assert!(requests.iter().all(|request| {
9710            request.model.provider == roder_api::catalog::PROVIDER_MOCK
9711                && request.model.model == "gpt-5.5"
9712        }));
9713        assert_eq!(
9714            requests[0].runtime.speed_policy.as_ref().map(|d| d.phase),
9715            Some(roder_api::inference::SpeedPolicyPhase::Orientation)
9716        );
9717        assert_eq!(requests[0].reasoning.level.as_deref(), Some(REASONING_HIGH));
9718        assert_eq!(
9719            requests[1].runtime.speed_policy.as_ref().map(|d| d.phase),
9720            Some(roder_api::inference::SpeedPolicyPhase::Execution)
9721        );
9722        assert_eq!(requests[1].reasoning.level.as_deref(), Some(REASONING_LOW));
9723        assert_eq!(
9724            requests[2].runtime.speed_policy.as_ref().map(|d| d.phase),
9725            Some(roder_api::inference::SpeedPolicyPhase::Verification)
9726        );
9727        assert_eq!(requests[2].reasoning.level.as_deref(), Some(REASONING_HIGH));
9728        assert_eq!(
9729            requests[2]
9730                .metadata
9731                .pointer("/speedPolicy/phase")
9732                .and_then(serde_json::Value::as_str),
9733            Some("verification")
9734        );
9735    }
9736
9737    #[tokio::test]
9738    async fn deadline_turn_timeout_emits_partial_result_and_clears_active_turn() {
9739        let mut builder = ExtensionRegistryBuilder::new();
9740        builder.inference_engine(Arc::new(DeadlineEngine));
9741        let runtime = Arc::new(
9742            Runtime::new(
9743                builder.build().unwrap(),
9744                RuntimeConfig {
9745                    default_provider: roder_api::catalog::PROVIDER_MOCK.to_string(),
9746                    default_model: "mock".to_string(),
9747                    runtime_profile: RuntimeProfile::Eval,
9748                    turn_deadline_seconds: Some(1),
9749                    ..RuntimeConfig::default()
9750                },
9751            )
9752            .unwrap(),
9753        );
9754        let mut rx = runtime.subscribe_events();
9755        let turn_id = runtime
9756            .start_turn(StartTurnRequest {
9757                thread_id: "thread-deadline".to_string(),
9758                message: "slow work".to_string(),
9759                images: Vec::new(),
9760                provider_override: None,
9761                model_override: None,
9762                reasoning_override: None,
9763                workspace: test_workspace(),
9764                instructions: InstructionBundle::default(),
9765                developer_context: None,
9766                task_ledger_required: false,
9767                service_tier_override: None,
9768            })
9769            .await
9770            .unwrap();
9771
9772        let mut saw_partial = false;
9773        let mut saw_deadline = false;
9774        let mut failed_kind = None;
9775        tokio::time::timeout(std::time::Duration::from_secs(5), async {
9776            loop {
9777                let envelope = rx.recv().await.unwrap();
9778                if envelope.turn_id.as_deref() != Some(&turn_id) {
9779                    continue;
9780                }
9781                match envelope.event {
9782                    RoderEvent::TurnPartialResult(event) => {
9783                        saw_partial = event.summary.contains("partial turn state");
9784                    }
9785                    RoderEvent::TurnDeadlineExceeded(event) => {
9786                        saw_deadline = event.partial_result.contains("transcript items");
9787                    }
9788                    RoderEvent::TurnFailed(event) => {
9789                        failed_kind = event.error_kind;
9790                        break;
9791                    }
9792                    _ => {}
9793                }
9794            }
9795        })
9796        .await
9797        .unwrap();
9798
9799        for _ in 0..20 {
9800            if !runtime.active_turns.read().await.contains_key(&turn_id) {
9801                break;
9802            }
9803            tokio::time::sleep(std::time::Duration::from_millis(10)).await;
9804        }
9805        assert!(saw_partial);
9806        assert!(saw_deadline);
9807        assert_eq!(failed_kind.as_deref(), Some("deadline_timeout"));
9808        assert!(!runtime.active_turns.read().await.contains_key(&turn_id));
9809    }
9810
9811    #[tokio::test]
9812    async fn deadline_skips_subagent_task_when_remaining_budget_is_too_low() {
9813        let calls = Arc::new(StdMutex::new(0));
9814        let mut builder = ExtensionRegistryBuilder::new();
9815        builder.inference_engine(Arc::new(CapturingEngine {
9816            request: Arc::new(StdMutex::new(None)),
9817        }));
9818        let task_tool = Arc::new(CountingTaskTool {
9819            calls: calls.clone(),
9820        });
9821        builder.tool_contributor(Arc::new(TestToolContributor { tool: task_tool }));
9822        let runtime = Arc::new(
9823            Runtime::new(
9824                builder.build().unwrap(),
9825                RuntimeConfig {
9826                    policy_mode: PolicyMode::Bypass,
9827                    agent_swarm_mode: false,
9828                    ultra_mode: false,
9829                    ..RuntimeConfig::default()
9830                },
9831            )
9832            .unwrap(),
9833        );
9834
9835        let result = runtime
9836            .route_tool_call(
9837                &"thread-deadline-task".to_string(),
9838                &"turn-deadline-task".to_string(),
9839                ToolCallCompleted {
9840                    id: "task-1".to_string(),
9841                    name: "task".to_string(),
9842                    arguments: serde_json::json!({
9843                        "description": "inspect",
9844                        "prompt": "read"
9845                    })
9846                    .to_string(),
9847                },
9848                None,
9849                Some(OffsetDateTime::now_utc() + Duration::seconds(1)),
9850                None,
9851            )
9852            .await
9853            .unwrap();
9854
9855        assert!(result.is_error);
9856        assert!(result.result.contains("deadline policy skipped"));
9857        assert_eq!(*calls.lock().unwrap(), 0);
9858    }
9859
9860    struct TestToolContributor {
9861        tool: Arc<dyn ToolExecutor>,
9862    }
9863
9864    impl ToolContributor for TestToolContributor {
9865        fn id(&self) -> String {
9866            "test-tool".to_string()
9867        }
9868
9869        fn contribute(&self, registry: &mut ToolRegistry) -> anyhow::Result<()> {
9870            registry.register(self.tool.clone())
9871        }
9872    }
9873
9874    #[tokio::test]
9875    async fn agent_swarm_mode_override_is_per_thread() {
9876        let runtime = Runtime::fake().unwrap();
9877        let trigger = roder_api::subagents::AgentSwarmModeTrigger::Manual;
9878
9879        // Off everywhere by default.
9880        assert!(
9881            !runtime
9882                .effective_agent_swarm_mode_for_thread("thread-a")
9883                .await
9884        );
9885        assert!(
9886            !runtime
9887                .effective_agent_swarm_mode_for_thread("thread-b")
9888                .await
9889        );
9890
9891        // Enabling on thread-a does not leak into thread-b.
9892        assert!(
9893            runtime
9894                .set_agent_swarm_mode_for_thread("thread-a", true, trigger)
9895                .await
9896        );
9897        assert!(
9898            runtime
9899                .effective_agent_swarm_mode_for_thread("thread-a")
9900                .await
9901        );
9902        assert!(
9903            !runtime
9904                .effective_agent_swarm_mode_for_thread("thread-b")
9905                .await
9906        );
9907
9908        // A per-thread `off` override wins over a runtime-global `on` default.
9909        runtime.set_agent_swarm_mode(true, trigger).await.unwrap();
9910        runtime
9911            .set_agent_swarm_mode_for_thread("thread-b", false, trigger)
9912            .await;
9913        assert!(
9914            !runtime
9915                .effective_agent_swarm_mode_for_thread("thread-b")
9916                .await,
9917            "explicit per-thread off overrides the global on default"
9918        );
9919        // A thread with no override still follows the global default.
9920        assert!(
9921            runtime
9922                .effective_agent_swarm_mode_for_thread("thread-c")
9923                .await,
9924            "threads without an override follow the runtime-global default"
9925        );
9926    }
9927
9928    #[tokio::test]
9929    async fn set_agent_swarm_mode_for_thread_emits_event_with_real_thread_id() {
9930        let runtime = Runtime::fake().unwrap();
9931        let mut events = runtime.subscribe_events();
9932        runtime
9933            .set_agent_swarm_mode_for_thread(
9934                "thread-xyz",
9935                true,
9936                roder_api::subagents::AgentSwarmModeTrigger::Task,
9937            )
9938            .await;
9939        let mut saw = false;
9940        for _ in 0..8 {
9941            let envelope = tokio::time::timeout(std::time::Duration::from_secs(2), events.recv())
9942                .await
9943                .unwrap()
9944                .unwrap();
9945            if let RoderEvent::AgentSwarmModeChanged(event) = envelope.event {
9946                assert_eq!(event.thread_id, "thread-xyz");
9947                assert!(event.enabled);
9948                assert_eq!(
9949                    event.trigger,
9950                    roder_api::subagents::AgentSwarmModeTrigger::Task
9951                );
9952                saw = true;
9953                break;
9954            }
9955        }
9956        assert!(
9957            saw,
9958            "expected an AgentSwarmModeChanged event for the thread"
9959        );
9960    }
9961
9962    #[tokio::test]
9963    async fn ultra_mode_override_is_per_thread() {
9964        let runtime = Runtime::fake().unwrap();
9965        let trigger = roder_api::subagents::UltraModeTrigger::Manual;
9966
9967        assert!(!runtime.effective_ultra_mode_for_thread("thread-a").await);
9968        assert!(!runtime.effective_ultra_mode_for_thread("thread-b").await);
9969
9970        assert!(
9971            runtime
9972                .set_ultra_mode_for_thread("thread-a", true, trigger)
9973                .await
9974        );
9975        assert!(runtime.effective_ultra_mode_for_thread("thread-a").await);
9976        assert!(!runtime.effective_ultra_mode_for_thread("thread-b").await);
9977
9978        runtime.set_ultra_mode(true, trigger).await.unwrap();
9979        runtime
9980            .set_ultra_mode_for_thread("thread-b", false, trigger)
9981            .await;
9982        assert!(
9983            !runtime.effective_ultra_mode_for_thread("thread-b").await,
9984            "explicit per-thread off overrides the global on default"
9985        );
9986        assert!(
9987            runtime.effective_ultra_mode_for_thread("thread-c").await,
9988            "threads without an override follow the runtime-global default"
9989        );
9990    }
9991
9992    #[tokio::test]
9993    async fn set_ultra_mode_for_thread_emits_event_with_real_thread_id() {
9994        let runtime = Runtime::fake().unwrap();
9995        let mut events = runtime.subscribe_events();
9996        runtime
9997            .set_ultra_mode_for_thread(
9998                "thread-ultra",
9999                true,
10000                roder_api::subagents::UltraModeTrigger::Task,
10001            )
10002            .await;
10003        let mut saw = false;
10004        for _ in 0..8 {
10005            let envelope = tokio::time::timeout(std::time::Duration::from_secs(2), events.recv())
10006                .await
10007                .unwrap()
10008                .unwrap();
10009            if let RoderEvent::UltraModeChanged(event) = envelope.event {
10010                assert_eq!(event.thread_id, "thread-ultra");
10011                assert!(event.enabled);
10012                assert_eq!(event.trigger, roder_api::subagents::UltraModeTrigger::Task);
10013                saw = true;
10014                break;
10015            }
10016        }
10017        assert!(saw, "expected an UltraModeChanged event for the thread");
10018    }
10019}
10020
10021#[cfg(test)]
10022#[path = "runtime/mailbox_test_helpers.rs"]
10023mod mailbox_test_helpers;